credit-part3-modeling-3-ensemble-tuning-part3.ipynb

{
 "cells": [
  {
   "attachments": {
    "8084db48-9d26-412d-a61a-28fff582b024.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAFnCAYAAAA2Qa+tAAAgAElEQVR4AeydB3gVVdrH/+mVFEJIaAkQOqFLEwUERRQQVKTqYkGF1VVZcBddC4gruLAg7lpRwf0UAWWpsiAg0nsJRUIJCRBCSCG9l/s97yQzzL0JkITk5pb/eZ6bOWfmlPf8ztybeeec9z0OBoPBAAYSIAESIAESIAESIAESIAESqAABxwrkYRYSIAESIAESIAESIAESIAESUAhQgeCNQAIkQAIkQAIkQAIkQAIkUGECVCAqjIoZSYAESIAESIAESIAESIAEqEDwHiABEiABEiABEiABEiABEqgwASoQFUbFjCRAAiRAAiRAAiRAAiRAAlQgeA+QAAmQAAmQAAmQAAmQAAlUmAAViAqjYkYSIAESIAESIAESIAESIAEqELwHSIAESIAESIAESIAESIAEKkyACkSFUTEjCZAACZAACZAACZAACZAAFQjeAyRAAiRAAiRAAiRAAiRAAhUmQAWiwqiYkQRIgARIgARIgARIgARIgAoE7wESIAESIAESIAESIAESIIEKE6ACUWFUzEgCJEACJEACJEACJEACJEAFgvcACZAACZAACZAACZAACZBAhQlQgagwKmYkARIgARIgARIgARIgARKgAsF7gARIgARIgARIgARIgARIoMIEqEBUGBUzkgAJkAAJkAAJkAAJkAAJUIHgPUACJEACJEACJEACJEACJFBhAlQgKoyKGUmABEiABEiABEiABEiABKhA8B4gARIgARIgARIgARIgARKoMAEqEBVGxYwkQAIkQAIkQAIkQAIkQAJUIHgPkAAJkAAJkAAJkAAJkAAJVJgAFYgKo2JGEiABEiABEiABEiABEiABKhC8B0iABEiABEiABEiABEiABCpMgApEhVExIwmQAAmQAAmQAAmQAAmQgDMRkAAJ2AeBJUuW2FxHO3fuDPkwkAAJkAAJkAAJmI8AFQjzsWZLJGBWAn5+fkhLSzNrm7Xd2LZt29C/f//aFoPtkwAJkAAJkIBNE+ASJpseXnbO3gi89tprCAwMhIODg90pDzLW9913H8LCwiAcNm7caG/Dz/6SAAmQAAmQgFkIOBgMBoNZWmIjJEACNUaguLgYo0aNwsqVK8ttIyAgAN7e3uVes+aTqampt1SULl68iJCQEGvuImUnARIgARIgAYsjQAXC4oaEApFA5QkMGTIEGzZsKFNw1apVyrmuXbva5IN0ZGQk5CMhOjoaf/7zn40YODs7Y9OmTRgwYIDReSZIgARIgARIgASqToAKRNXZsSQJWASBmJgYNGvWTJOlW7duGDlyJKZPn66ds6eI2EDs2rULRUVFSrf/+te/Ys6cOfaEgH0lARIgARIggRolQAWiRvGychKoeQIzZszAzJkzlYZatWqFY8eOwcPDo+YbtuAWTpw4gY4dO2oScqWmhoIREiABEiABErhjAjSivmOErIAELIeAKA72rjzIaHTo0MFyBoWSkAAJkAAJkICNEaACYWMDyu6QAAmQAAmQAAmQAAmQQE0SoAJRk3RZNwnYHYFc7Di3CLOi9tym50U4dWU1Zp1ehbhSW4XbFOBlEiABEiABEiABCyFABcJCBoJikIC5CbyzezzC1g5Dn99m4GyJvXGJCMXZmL6z9Nr+5ZUUKx8RsWux5MrJ25QrQnTiDiyJ2Y5kg75xKWZAdkEadl5YhLbrHsHQQ0tvUxcvkwAJkAAJkAAJmJMAFQhz0mZbJGBBBNLz0+GAQGRkHcbsmIuaZAlX/oU1GUVoDCA+rxZ2si5MxYxD87A+PReODg6aXIyQAAmQAAmQAAlYBgEqEJYxDpSCBGqFgJtje4T71MW5Sz8jsRhAcSq+jzqEBgHh6KKXqCgV6yI/Qpt1jyizFg/+OhObkq+W5ChIwOwDL6Hl2mEYsPtTxBbqC2Zh54Wv8VBpuYd+nYV9Gen6DGXjzv74R+9Z+LDVALg48CeqLCCeIQESIAESIIHaJcD/zrXLn62TQK0ScHB0wQN1m+Na7jFczM5EVlYUfsjMR696nXVyFWDH2YV4/exejOr8Idbd/Tc0yjyEqQcX4STk2r/wVXwWpnWZizeCHPFTzo2ip2M+xeSTv6B7abnuxQcw+eSaGxkYIwESIAESIAESsDoCVCCsbsgoMAlUJwEn9GnQFV4F1/FT8kUcvLYNqcUe6FOv9Y1GCjPwU0IM/Py64q8NW6NdvV54q1075ORH4sj1DPyUFItAv7Z4smErPNBiEl6rU1rUkIENsRfg4NEAbYpjcTo7EwF1ApCeeBDbjGYpbjRVU7G1a9fWVNWslwRIgARIgATsjoCz3fWYHSYBEjAiUKduH4z0+hKLz21Dtut2FNUZi7t9PbBZzWUoRFJxITxd3eBYuqSonltdAL8jOa/sNbUYiotwvbgA2TmxeDviY+00EAgvZyBLd6amo++88w4eeeSRmm6G9ZMACZAACZCAXRDgDIRdDDM7acsE1F2oq97HuhjStCdcszfh59Q6mNrlUfjrK3Pyxn2evojPuIy9OQVAYTw+iY4AnNuhb4AHurh4KdfO5BQgOe0wtueWFnZyRwcPL7i5hmLRvd8g6pF1OPPAR3izy1j00NdvhnhERARGjhxphpbYBAmQAAmQAAnYPgEqELY/xuyhjRLIzMyEj49PtfQuNGgwGjo7wsmrBZ6s42Jcp6Mnnm/1EHwMV/D81scQtuF5/CfDAePajEU31zqYENYLHkXReHzrY7h39xe4YnAqLe+OMeFj0RpJeGnns4rxdeutr+PzuNPG9ZumilLxtx0TEbZlOjKKCnE67geErX0Mn8fHmOasVHrlypUYMWJEpcowMwmQAAmQAAmQQFkCDgaDwVD2NM+QAAlYOoElS5bgmWeeMRKzU6dOOHbsmNG5myVOJO5DQnF99AlqDtfiPBxMPokCRz/0rhsGJ4ccnLgagasuwRhUr6lSRUpWLA6nx0KcNfl5hKKjTwO4l76CiE05idO5majr1RgtHa5jX56PVi4zJxa/p8Ui1QA4unijg38bBDk5Ij7tFI7nAr0C28PHUfcuw1CA35NPIrYgTye6A1r6d0Azd0/duVtHHcpxARscHIzz58/Dy8vr1oV5lQRIgARIgARI4KYEqEDcFA0vkIBlEyjvAbkyCoRl9+7OpSuPj1or35uoJHgkARIgARIggcoT0L32q3xhliABEqgdAqNHj66dhq201WXLlhlJ/vnnnxulmSABEiABEiABEqg4ASoQFWfFnCRgEQRkidKKFSssQhZrEUIUrn79+mniTp48GUuXLtXSjJAACZAACZAACVScABWIirNiThKwCAJduhjtEY13333XIuSydCF+++03IxH/+Mc/GqWZIAESIAESIAESqBgBKhAV48RcJGARBJ5++mkjOVJSUozSTNyagBhQu7u7K5nS0tIwaNCgWxfgVRIgARIgARIggTIEqECUQcITJGCZBFJTU/Htt99qwk2dOhV+fn5ampHbEwgLC0PLli21jJs3b0ZcXJyWZoQESIAESIAESOD2BKhA3J4Rc5CARRAYP368kRzz5s0zSjNRMQLHjx+H3gi9bdu2OHnyZMUKMxcJkAAJkAAJkACoQPAmIAErIDBjxgxs2LBBk/To0aNaXB+RzeXkY88hKSkJpvYOpjwGDx6snUpPT0eHDh20NCMkQAIkQAIkQAK3JkAF4tZ8eJUELIKA3u1oq1at0Lp1a02uSZMmafGoqCh069YN77zzjnbO3iLdu3fHfffdp3V75syZWlyNiC3Jjh071KRynDJlilGaCRIgARIgARIggfIJcCO58rnwLAlYDAFxN6pfvlTeJmjjxo3DDz/8UEbm4cOHY8SIEUbnW7RogXvuucfonLUlZBduffjqq6+we/du/Skl7uLigl9//fWm/RVFQm9X8sknn4Demcpg5AkSIAESIAESMCJABcIIBxMkYFkE5EH5mWee0YRavHgxTD0xqRdNH4bV8/Z6FPe2svTrdsF0x+ryFLTb1cHrJEACJEACJGBPBLiEyZ5Gm321OgL6zc7E/WiPHj1u2gdRNhISErBgwYKb5rGHC+Hh4QqDiigPwkOYiXcmNfTq1UuN8kgCJEACJEACJFAOAc5AlAOFp0jAEgh07twZERERmihVeTP+0UcfQT76IEbWycnJ+lNWFQ8KCtL2ctALHhMTo09WKj5mzBgsX75cK3Px4kWEhIRoaUZIgARIgARIgARuEKACcYMFYyRgMQSio6MhCoR4CJIgbkeXLVtWLfJdunQJR44cqZa6aqMSsd+oV69etTf9xhtvYM6cOUq9zs7O2LRpEwYMGFDt7bBCEiABEiABErB2As7W3gHKTwK2SGDr1q2a8iD907sdvdP+ypt1vl0vS3HIkCGYO3cuioqKUFhYiIEDB6Iqsz5la+YZEiABEiABErAtArSBsK3xZG9sgIC4YH3++ee1noi70ZsZTmuZGLljAjKzYbq/hn7DuTtugBWQAAmQAAmQgI0QoAJhIwPJbtgGgfz8fMXtqNobMe6VfR0YzENANpQT701qWLFiBaZPn64meSQBEiABEiABEgBAGwjeBiRgQQRSU1Ph7++vScQlNBoKs0bo2tWsuNkYCZAACZCAlRHgDISVDRjFtV0CGzduNFIepk2bZrudtfCe5ebmQna0VoPMBOXk5KhJHkmABEiABEjArglQgbDr4WfnLYmAKBBqkB2U9RvIqed5NA8BNzc39O3bV2vswoULOHfunJZmhARIgARIgATsmQAVCHseffbdYgiMHz8eCxcu1OQRF6Lt2rXT0oyYn8C8efMwf/58reFOnTpVmytdrVJGSIAESIAESMAKCdAGwgoHjSLbFoH4+Hi0aNECWVlZSsdat26NyMhI2+qklfbm/PnzuOuuu5CWlqb1gHYpGgpGSIAESIAE7JQAZyDsdODZbcshIEuXVOVBpKLXH8sZG1HsDh06ZCTQvffea5RmggRIgARIgATsjQAVCHsbcfbXogisXbvWyNZh8uTJ3PPBokYIyuzQqlWrNKl27dqFJUuWaGlGSIAESIAESMDeCHAJk72NOPtrUQQ6d+6MiIgITSYuj9FQWFzEz8+PS5ksblQoEAmQAAmQQG0Q4AxEbVBnmyQA4NixY0bKw969e8nFggnIHh36EBgYCLGRYCABEiABEiABeyNABcLeRpz9tQgCSUlJ6NKliyZLkyZN0KtXLy3NiGUSWLBggSaYjKGpfYR2kRESIAESIAESsGECVCBseHDZNcslYLpJ3PHjxy1XWEqmEXjttdfw3XffaemxY8di9uzZWpoREiABEiABErAHArSBsIdRZh8tisD69esxbNgwTSZ5AKXnJQ2HxUcyMzMVw+pr165psqakpEBsJBhIgARIgARIwB4IUIGwh1FmHy2KgIODg5E8NJw2wmEViUuXLiE0NFSTNTw8HCdOnNDSjJAACZAACZCALRPgEiZbHl32zeIIzJgxw0imo0ePGqWZsA4CISEh0I/dyZMnYTq21tETSkkCJEACJEAClSfAGYjKM2MJEqgyAf3sQ6dOnRRPTFWujAVrnUD//v2xfft2TQ7OJmkoGCEBEiABErBhAlQgbHhw2TXLIsB9BCxrPKpLGr1S6Orqip07d6JHjx7VVT3rIQESIAESIAGLI8AlTBY3JBTIFgnIzsVpaWlK1+SB88cff7TFbtplnzZt2gRVicjPz8eOHTvskgM7TQIkQAIkYD8EOANhP2PNntYiga5du2pr5n19fWG6KVktisamq4HAzz//jKFDh2o1ibtX/Z4R2gVGSIAESIAESMAGCFCBsIFBZBcsm0Dnzp2NdpzmOnnLHq+qSte0aVNcvHhRK75t2zaIjQQDCZAACZAACdgaAS5hsrURZX8sisCZM2cgHzW8+OKLapRHGyOwb98+ox5NnDjRKM0ECZAACZAACdgKAc5A2MpIsh8WSUBdG68Kx9kHlYRtHmVpmr+/v9a54cOHY/Xq1VqaERIgARIgARKwBQKcgbCFUWQfLJLA008/bSSX7FbMYNsExNPWhAkTtE6uWbOG9i4aDUZIgARIgARshQAVCFsZSfbDogjk5OTgwIEDmkzdu3eHh4eHlmbEdgmIxy19kBmJjRs36k8xTgIkQAIkQAJWTYAKhFUPH4W3VAKenp44ffq0Jp4oE25ublqaEdsmcOrUKaPxpgJh2+PN3pEACZCAvRGgAmFvI87+1jiBZcuWGbUxf/58ozQTtk+gXbt22LJli9bRhQsXYvz48VqaERIgARIgARKwZgJUIKx59Ci7RRLQv23u1KkTpkyZYpFyUqiaJXDPPfegffv2WiNLly6F6fIm7SIjJEACJEACJGBFBOiFyYoGi6JaPoF7770Xu3bt0gSNjIxE69attTQj9kVAdh9v3LgxMjMzlY43adIEly5dsi8I7C0JkAAJkIDNEeAMhM0NKTtUWwTOnz+Pw4cPa8336dOHyoNGwz4jsuv4U089pXX+8uXLkI0FGUiABEiABEjAmglwBsKaR4+yWxQB/Z4P8uAoewIwkIAQmDFjBmbOnKnB4C7VGgpGSIAESIAErJAAZyCscNAosuURmDt3rpFQs2fPNkozYd8ERIHQh/vuuw9ff/21/hTjJEACJEACJGA1BDgDYTVDRUEtlUBubi66du2quW3l7IOljlTtypWeno6GDRsiKytLEeSBBx5Q9odwdOR7nNodGbZOAiRAAiRQWQL8z1VZYsxPAiYE7r//fk15kEv79u0zycEkCQA+Pj7YsWOHhmLz5s0YNmyYlmaEBEiABEiABKyFAGcgrGWkKKdFEhAvS23bttVkE8NpvRcm7QIjJFBK4K677jIyto+OjkbTpk3JhwRIgARIgASshgBnIKxmqCioJRKYM2eOJpa8Td6+fbuWZoQEyiOwc+dOdOvWTbvUq1cvLc4ICZAACZAACVgDASoQ1jBKlNEiCXz66af49ttvNdkeeeQRODk5aWlGSKA8Ah4eHhg5cqR26dq1a/Dz89PSjJAACZAACZCApRPgEiZLHyHKZ7EE9G5bRUiDwWCxslIwyyOwevVqPProo5pgixcvxtNPP62lGSEBEiABEiABSyXAGQhLHRnKZdEE9MtOwsLCkJCQYNHyUjjLIzBixAiIxy41PPPMMzB1B6xe45EESIAESIAELIkAFQhLGg3KYhUELl26hP3792uyilFsYGCglmaEBCpKQDYb1CsRMgshboEZSIAESIAESMCSCXAJkyWPDmWzSAKyXj0tLU2TjUuXNBSMVIEAPXlVARqLkAAJkAAJ1CoB51ptnY2TgJURmD59uqY8iMH0+vXrraYHv/32m9XIWlVBO3fubHUGyW3atIG4/929e7fSbTmePHkS4eHhVcXAciRAAiRAAiRQowQ4A1GjeFm5rRHQG05bw47T8kCthoiICDVqs0exR/H29tb6d+zYMS1uyZGioiLFoHrdunWKmLLp3OnTp5Wdqy1ZbspGAiRAAiRgnwQ4A2Gf485eV4GA/mG8U6dOsMSH0yVLlig9E4NcewxRUVFG3VYVPrEtGDx4MIKDg42uW0pCZrPEDbCqQKSnp6NRo0b07GUpA0Q5SIAESIAEjAjQiNoIBxMkUD4BWf6jf4MvHnQsLYgLUFEc7FV5uNV4CJMGDRrAEsdNlXvixIk4evSomlSOM2bMMEozQQIkQAIkQAKWQIBLmCxhFCiDxRNQ32Srglqa4fTkyZPx+eefq+JpxwULFmjx1157TYvbcuSjjz5Surdx40Zs2rSpTFcnTZqEzz77rMx5SznRv39/ox3Nxe5m9uzZliIe5SABEiABEiABUIHgTUACtyBQXFyMUaNGYeXKlUouLy8vxMXFQdaoW0qQt9QzZ87UxJE37ePGjcO8efO0c/YcmTZtGpYuXYqrV69qGFJSUiza2Lp+/fpITExU5K1Tp45yz+ltO7SOMEICJEACJEACtUCAS5hqATqbtB4CshZdVR5E6pEjR1qU8mBKsmnTporxLZWHG2SEhRgkt2/fXjupzlJoJywsIsvl1L1FMjIy0LdvXwuTkOKQAAmQAAnYMwEqEPY8+uz7bQn4+/sb5VGNlI1OWlBCPEPpNyazINFqVRRhUq9evVqVoTKNyyzSgAEDtCJiG3Ho0CEtzQgJkAAJkAAJ1CYBKhC1SZ9tWzQBU5uCZcuWWbS8FM62CMj99uKLL2qduvfee6lEaDQYIQESIAESqE0CVCBqkz7btmgCYpishn79+mH06NFqkkcSMAsBcT2rhtzcXHTv3l1N8kgCJEACJEACtUaACkStoWfDlkzAz8/PSDx72MXZqMN3lCjGb5F/R9j/JmJfRvotazob/yParR2GmTG/3zKfvV4Ut7Ni8K0PluyKVi8n4yRAAiRAArZLgAqE7Y4te1ZFAgcOHEBOTo5WWjYhs9Ww/dg4hK0dhlbrR2JOrG4TtsJ4zN5Rem3jCzicmWV2BOvP/BMPb3pKka/F2j/gqYPLcdHsUtR+g6LMTpgwQRNkzZo1ePbZZ7U0IyRAAiRAAiRgbgLcidrcxNmexRPo2bOnJqMY38oGbbYaMgoyAAShjXcSFkXvxZ8bh8EVQGLKQXyVloWmrr6IMRSjwFBUgqA4D4l56cgzFMPJwQU+rr7wcnJSrhUWpSMxLwfFcEZ+scEEWSFSc1KQWVqurltduN3m9UVyvj8W3r8Y9XOjMPHXadhz9TusSBiJ1+uXtGfSgE0nxXj/l19+0VzRilI7f/58i3ZFW9kBiYmJqWwR5r8JAfHGxkACJEACNUmACkRN0mXdVkdgypQpRjJ///33RmnbTPhjeNNWOH98I7ZmPYmHvIB9icfh6tgIPX19EJOaVNLtwgT88/B8LEs+i+uFBXB39EbH+n2woMvLCC6+hJcOvY8dyVdR4OCNpop24FGKKw0/HF6IRdeO4WJpuT6NRmFOx0dviXNCh9K37F4N0NcbOJIB5BcXALA/BUJAbdiwAV26dNGY9erVC5GRkVraWiPqkiyZWWGoHgLDhw9Hs2bNoN9IsnpqZi0kQAIkUELgNu8AiYkE7IeA2Dno9wd49913MWTIELsA0CTwfjRyycDGuJMoRgZ+SzwJv8ARaOV+42H9RPxGfHXtFO4Jm4aoYauxoEk4DsTvxLKUiziR+JuiPDzR+n1EPvwPdFV1BwAxCTvxftxBdCott6RFOLbGrcWRzOsVYpuRcRL/zQC83e/GY8HuFSpji5k6d+6MV199VevamTNnsGvXLi1tbZF9+/ZBdngXxYHKQ/WOnvCU3zLhK79rYoDPQAIkQALVSYAKRHXSZF1WTWDixIlG8k+aNMkobQ0JeaiUT2WDr1tTdPSsi0Pxm5Gb+CP+l+aAcS3vhU4PQFTqaeQDGNSgJ+DghAFNOgPIwe9pGdq13g3C4ezUCA8HttNEiEk/jQIDcCD2ewzdPgVvx0YCRRnYnWFsHKwVMInMP/4JLjm44k8dx6OtyTV7S8pDoSxdUoN4B1u7dq2atJrjtm3buDmemUbrvvvuQ9euXc3UGpshARKwFwJUIOxlpNnPWxKQN7lRUTeMiMeOHYvg4OBblrGEi0uXLsXMmTM1UeRNo94AXLtwm4iDsw9G1m+C+PSTmHp6Ldx9umKsv4tRKScHJzgAKCy1hyg0FAJwgIujI0yv6Qs6OZSslGzgFYq2vs3RIaAbHmsyEPf6Gm/Spy8j8YL8RPxj9yT8J90DwzvNw8TgENMsVU4LM09PT5w/f77KddRWQVEavLy8lOaLi4shy1WsKfTv31/ZJK+gQJajlYQ+ffrg7bffhsFg4OcOGYjBvd7oXgjLTuyDBg1ScfNIAiRAAndMgArEHSNkBdZOIDU1FbJJlxrkn688mFt6EMPa8ePHG4nZqVMnyFKXygdn9A4diEZFCfgltQh9G/SB6b7NrQM6KQbWP108DBjy8UXUNsDRFw8EBKJBnRCImrD14mEUFJ7H17H7NBGCvBor17ycQzG3y2uY22YYfIvj4amc1bKViXwd8Vd8kXwFf+g0A/NDmpW5fqcnRNFq2bJlFXndaetVLy9vkzMzM40qqNqYG1VhlsSxY8ewfft2o7ZEaRAF/r333jM6z0TVCMjvgnyEqyibati8ebOypElN81hCQJZ52cNHvzyXY08C1UGARtTVQZF1WDWBN954w0j+cePGGaUtLXHlyhXloTcpqdS4uboE9OyOB+vXxzcJCehVL7xMra0CBuBvrRLxSdQChMXkwNejGV5s+wru9wmEq8cjeCn4PL6K+QDh14Iw2q899pTK1yrwQfyzVRK+vLgSYWu/A5y90DtkNCa4uSHnpt5hf8evVxMVGf5zaDL+UypNaMAzWNPnMdQpI13VT0RERGDkyJF46KGH8Nxzz1W9IjOXFBsddfZJ+iBr3eXtvqUGUcpNFV7ur1KzoyV85Z7+5ptvtIZEqTBV4rSLdhC5fv26oqwuXLjQDnp7o4viIEQ+Q4cOhTgusKbfuhu9qFps1apVVn3PW6ry52CQ1xQMJGCnBGT2wd//xlKao0ePWuwbaVmucunSJcW7ys2GS2Yg5C0vQ1kC8nCtPjiJe960tLQymeRBvGPHjmXOW+qJkJAQXL58WRPv2rVrqF+/vpa2lIjcu6NGjcLKlSsVkTw8PHD27Fk0btzYUkS0aTnuuece7N69W+ujvIRo2LChlraXiCgPAQEB9tLd2/YzOTkZdevWvW0+a8wQGxuLL7/8ErNmzbJG8cuVOTo6Wvl9l+W3lhA4A2EJo0AZao2AuMJUQ/v27W/5cK7mq63jsGHDFFee+va/++47rFu3DsuXL9efZvw2BF577TVFUfz4448hBr1qEAVMbApWr16tnrLo4/Hjx40UYJFflCBLUyLS09M15UGAynp8Kg/mu7VkiZgs01HDww8/bHcvGuSt+4kTJ1QE2lHeyNvD2/hHHy3rNrt79+5Gtn8aFCuPnDx5Ej169KiSPaAld11cM0u/HnjgAbz//vu1LioViFofAg+4fEMAACAASURBVApQWwTkn6reY9H9998PeTNtaUHk/Oqrr8ooD7Nnz1aWhJw7d87SRLYKeeSBQj4yMyGzNuqMhLjA9Pb2xvr16y16SZBAll2q5T5Ql+HFx8fj119/xZgxYyxqDOQe1gdrUdD0Mlt7XPaEUPe5ERsamX2V+8cegryNNnUVLEu5li1bZhXOMqpjjGSxifw+yG+DOhN74cIFRbGUWUxbUOjFiYi4hxbPY6ZBbw9kes2S03Lv6h28HDhwAPKRzSJNPUeavR+yhImBBOyNwJo1awyOjo6yfE/5zJw50yIRmMop8rZq1cpw9OhRTd53331X60enTp2084wYE+jXr5/GSZjpw7lz57Rr6j0hx1GjRumzWWx82bJlRvJ/9tlnFiWrnumiRYssSjZ7ESYhIcHoHtm2bZu9dN3QsGFDo77b+z04f/58Ix7CxxZCdHS0Ub/U3x39/0tr66d8b0X+cePGlenb22+/XavdoRcms6tsbNASCIjvfFmXLcHFxUVxK2kJcqkyyFpd8aQiy2lUOeWauLuUWRNr8bqj9sfSjy1atFC81kyePNlI1BUrVihv6OStviWH3r17IygoSBPRtB/aBQuICGsG8xMIDAw0f6MW0KK8qY2Li9MkmTp1au2/udWkqZ2IzEQJBzUIH+FkzUHeyssSH334/vvvld91a/5/Kd9bkV/6kpiYiCZNmmhdFPsOeU6orUAForbIs91aJfD1119r7YtBkhgZWlIQQ79nnnnGSCTV3aXRSSaqlcCnn36q/MMxrXTgwIFGa8hNr9d2WoypZXmCPtjL8hR9nxknAT0BWSp38eJF7ZQs45o3b56WtueIcBAeahBO1ry0sGfPnmpXlKMYHFu6R0UjgSuQqFevnuJIRZ9VnhNqy0sTFQj9SDBu8wRk4zD9m7gHH3xQWQtsKR0X49527W7s4qzKtXfvXjXKoxkIyFpa/T9XtUkxulftDdRzlnQUd4VqEJuO2nw7pcrBIwlYCgFrXQdfU/xshYf839SHH3/80epnVPT9MY2LzYrei5r+d980b02mqUDUJF3WbXEEDh06BP3+CYMHD7YYGeVHUHyTy66xapBpZpl50HuLUq+pR/2bZvHAs2HDBqM+qvns8SjLv2JiYpTZA9VwUDjomZXHxc3NDTIep06dQmhoqJZl//79mDNnjvIWU78sQstQyxExCpdlbmqQt1OWKKcqH48kUJME9J6H5HvcpUuXmmzO6uoWHvrfNz0va+qMfk8PeUEo+/rYchCDd/G4p4YdO3aoUbMeuQ+EWXGzsdomoHdlKC5QTTe2qg35ZH29uBM19RIyf/58zWvK7eTS90vyyjpJ2bFYgjVPS9+u37e6Lg/TBQUFZbxXSZnKbn8jsw6iOOiDj4+P4ls/PLzspnv6fLUR198PliCnXh5xm2vJG97VxniZq017Gwd9fyv7nTfXmFhCO9bMSdyYP/LIIxpGeRsvv/32EPTj9vzzzyv7Xpiz33Tjak7abKvWCBQWFio7cKoCiMGpGCjXdpAZEVlfrw+tW7dGZGSk/tRt4xkZGRDjVNlITIJMcaobjOl/ZNSK5EdXXJXaQijPZd/N+iVT9uKetbJBXKXKRwwN1TXVsrdBhw4dlDf+suOvs7Pl/JwuXrxYs6ERObds2QJLVHQqOw7MTwIkQAJ6AnoXpzKbYi/KgzDQ/87/8ssveixmiVvOfzyzdJeN2CsB8Xu+adMmrfuydKm2H6DFs4J+HwoRTmTas2ePJmdFI1JOvFCIrURF9gCQTensLSxatOiOva+Ij3HZd6Nv374aPtnhV5YCvPTSS5g0aZJ2vjYjTz/9NMLCwjQ5xeuKKD7l2XXUppxsmwRIgARIwDoJ0AbCOseNUleSgL+/v1ZCHp5r07g0Pz9f8fok9gpirKsGmXqVmYS6deuqpyp1FE88o0ePVpbn7Ny5ExMmTDBaJ1mpymwgs6wRFQbCQpYvVMemO8HBwbj33nshyoheCZOdT8V1qtxXYqhvCaFbt26KEqHKIp46xB6EgQRIgARIgATulABnIO6UIMtbPIEZM2YYyfjee+8Zpc2ZEFlmzpxZpsmUlJTbGvaWKXSLE+KW9lauacWIWN15+RbVWMWl2ljbLMqIfGQtv944W3W9WxsymQ6WuCcWZUa/hE38pFuCbKay2k46H0fOTMcTUbn4bsBc9Hb3umnXLl/7Fv33/4TXe/wLk4Kt2wf/TTvJCyRAAjZLgDMQNju07JhKQP/ALmvga2tTGTHE1csi8k2bNk15oLudVyC1L9V1TE1NVdqVh0lr/1QXk6rUI7YP8pDu7u5uVFwe2i3FE8jRo0eNZDNVqI0uWliiqCgHk3f+AWFrh+Ph/euQUSrfgZiP0XL9KGxNy6oRieOTNmHWyeWIzK9s9cWIzykCiguRYSjZqPJmNaQXlFSeU1Rwsyw8TwIkQAIWS4AKhMUODQW7UwJi9yAeaNQgm7PJA5+5g7jRlAdKvRcfkeWhhx7C3LlzzS0O26tmAmJrkJOTU8a+YOXKlcq4Hz9+vJpbrFx1ojDL0jY1iBJ76dIlNWnhRwNQXABP76a4mrAIS5NKHrbzikzFLkJGbhKuZF9DXE4yckuf3XPykxGbnYSc0nRanuRJQYFByhcjIzcZcTkpKDIoJ5RK8wvTEZm4A0su/IKjmdcQm5eD/MJMxGZfQ2Z+BmKzE5FeVIii4jwk5iYo52Nzk5Gn7Gzvhv7t3sX2AbPR191LK5dVmIX4nGuKLJmFJcK3DB6L7fd/hYnBzVBUnIv47GtIKchDklJnIq4XFGqdLJTy2VL+xidHaU/LwggJkAAJmJUAlzCZFTcbMyeBn376SbEpUNusjR1IZTdM0xkPWVpy5MgRiM0Cg+0QkH0jxEuTuOQVV6VqEFsM8fhVm+50ly1bhuXLl6siKbYR4lRgwIAB2jlLjgR53YsezlnYdHE9/lD3URNRc7Hx9CJ8FLMd5wvy4OLgjs71h2Fetz/gWvQ8PHHmHD7o9xNG+wLzD7yAZWk+WNz/c9ztno0390/DITTD+j5vIcDZQan3fOw3eO5cidL31q6JQNDLWN0oHiOO/IQ2XsGIzMrEtLvfgtPFb7Ao4TyuFxQDzp54sNlL+LRtL/z6+/t4Na5kCZN30kqlXFvfEFzKvISsIjf0bvQoPu88HtHxJddkCdMT3lcxdvsHKPJshdzc80gucEQD7/ZY0ud9hDom4++HZ+GnhCjk6Hr+Zb9lGOh78yVSuqyMkgAJkEC1E7BrBUJcaMpbaoaaIVDbvt7V9ejSuyFDhkA805grqG5j9Z6fpG1ZQlUbsyDm6re9tyMuBOUj9/6xY8c0OxPZ40M8ZYkL2dr6Xogx+f3334+8vDzI/Slu/6xFgXB08sVLbfpj6JGtOJ3zoNFtlpp1Dv+M+QV1g57D8c4jcOXyNxh8fA22Xh+IBxsMRIOzJ7Hl2ik84Z2JwykFKEQ6DqdfQWcnICYvHaGBveHtfGMyvl3T17CuKA3DTl3BZw98iUEewInYb5U2U+vcj30DRyMQqfjP1TB812c2WntkYM7Wp7Ho8h5cadvLSDY1kel1H7beOxjfH/gL/pV0CNcKH1cvGR0zHALxf/fPRkLMv/H86W04mB4Pd6ed+DnhMl7s9R3+VCcCg7bMRavOP2Ggr5tRWSZIgARIwJwE7FKBEJeG8oZQXGjqveCYE7w9tKXulCgPUuYO+iUb0nZVfP/ficzi1lM88+iD7BYpHnwYbJ+AahvRsmVLrbNZWVmQPStGjRplNBugZajhiBjVy8xX+/btlZY+/PBDyAyZfmaihkW4o+rr1h+J5/22YtLZw/jnjZWJSM04iIQCIDX5F4za8SuKijIBQyGOpcdhlE9r1HN3xdXrRxHtdhXnPILQL+caDqXG4hGni4jNzcfwwI6o6KP4H5v2RaDSCz8Ma9od885OR0RGITJvrIAqt49jGt+FQEdv9PX3xb+upyHBUITydmEJr9cXrV1c4esTBmAb0opyAUW5KS5ZZmUogiyAcnQomS0ptzGeJAESIAEzELjx2sUMjdV2E+InX9aiiztDUxeatS2bLbYvjOUjzMXz0a5du8zSTXl4W7FihdbWu+++q8VrOrJ06VKlv6bKg7j9pPJQ0/Qtq37Z2E8M1MW9qz7IvSnfCdmB3NyhXbt2Rq59Zd+Q+Ph4c4tRxfbcMaTF4yiMXYUDOrtjB0cXyD+yAM8QtPVtjvC6HfFYk4Ho798Q7q7+6O8dhKs5R7DlyjG08uuLp5p4YVfcQRxMOoR0p3YYWj+owvK4O5a+c8s9jhd3v4cN6d5Km01cKlxFpTP6uLVAgKsDvjn0FMK2zIej70BMauxa6XpYgARIgASqk4BdKRA9e/asTnasqxIE5CFeHqDF+09NB9Odic3ldUb2ABg/frxR92QWRh4iq2MPAqOKmbAaAp9++qlyD5gKLDuQiyJh7qCfEZTdyhs0aGBuEarcXmi9u/GEewKWxxzR6vD2bAE/JwBFXninw8uY2340QoovocDgCshb/7q+SM04h38kpaONf0d0D30Eftnb8deoaDRr9CDuuumz+FVcvZ6ktWMUyYrE4Xzg7maPY277IUjPNrparYnYq6txLr89vhuyDlGPrMPmvq+hXbW2wMpIgARIoPIE7EKBkAc703/U4jlFdmW1dheWliq/vNkUvuHh4UZ3pWzotnHjRqNz1ZmQGQB9kP0VajqIG8/AwEDobS6kzbFjxyrr4Gu6fdZvHQRkuWR5O0H36tUL4uLXnOGDDz4wak7sNqwj1MXLPf6AusXJyqyDu5PMPHTF++1Gwz9/L7pteBRhm1/BDueuaO1VYmDcNeRhqLsstPFtDnef+9He0w1w8MYTDVuV2+2QBiNwt5sH3jv8DHodK2dn+ICBeLtRJ2w9/hbCNs9E+/otyq2nOk76+XdFoMMxPLp2GMLWDkP79c/ghSP/RUIZT1TV0RrrIAESIIEKEjDYeLhy5YqsTtU+Tk5Ohk2bNtl4ry2re3v37jW4uLhoYyDjcfHixWoXMiUlxaiNCRMmVHsb+gqLiooM0dHRRm1K33x9ffXZGCcBIwKnTp0yhIaGlrlv5s6da5DfK3OFqVOnGsmwc+fOGmta/xu8bdu2GmvHFiv+37GJhuZr3zJEZJf0bu7hVwzN1ww1HMyqfG/tbRz0/a08LfspYc2cFixYoP2Oye+qPYXFixfXat9tegZC1qG3bdvWSJXasGEDBg0aZHSOiZolIG9YTb0RyQzQ/v37q7Vh0+VDr7zySrXWb1rZsGHDIDv7moZ9+/aZnmKaBDQCYocQExOD6dOna+ck8vrrryu/V6b2M0aZqjFh6tZYlhiKu1cGyyJQUJwPOEZj9rH3MenA+9iYeBnwuA+NKmr5bVndoTQkQAI2QsCmFYgtW7YgPT1dG6off/yRyoNGw7wRsUv43//+BycnWawMxY3k5s2bq00IsXMQ5VANsvtu165d1WS1HsUYXFzC6tuTBmbPnq0siWvTpk21tsfKbJOAer+EhoZqHZTfqw4dOkA8JpnDQ9y5c+fg6+urtV+Tywu1RhipFIHBHT7C1Ob3wqE4C+kFWegS9iJ2PfBnNCj5Ka1UXcxMAiRAAtVFwEGme6qrMkurx9TuwYa7amnobyqPn5+f5htfMlXXmAQHB+PatWtKu926dYP4vPfw8LipHFW9sHbtWjz66KMoNtkFVt7cmrqOrWobLGdfBMQLkjzI9+3b16jjMnsqs2iTJk0yOl/dCbHh0bubFeVFvj/VGfS/xeJCu7b2wqjOPlljXfY2Dvr+Vtf/Gmsc99vJbM2cxKumuOaXIC9jZHbXXoLY96q2l7XRd5udgTB1kWgOY1p7uWnvpJ+mXpiq442nGE6ryoPIJhsEVrfyIA958mWVHYX1ykOfPn0UJYjKw53cFfZdVpRfWT4krn5lWZwaTp8+rbiAlftOHvJrKoi7Wb3nssOHD9doezXVD9ZLAiRAAiRgPgI2q0B8/PHHGsUJEyZA3nwzWAYB/b4Mc+bMuSOh5OFKb/uwePHiO6rvZoXF1aWq6at5ZOmHufa2UNvk0XYJiKtfmeGS3cr1Qe47/QyB/lp1xeWFi7qUKicnp8bbqy65WQ8JkAAJkEDtELBZBaJ2cLJVcxPQu211d3dHjx49qlWE1157DWL0ahrETa3pbIppHqZJoCoE1F2sxeWxPsgyg5EjR+pPVWv8nXfeMaqvc+fORmkmSIAESIAESEAlQAVCJcGj1RGIi4uD3hBb3pyW97Bf1Y6J8rBw4ULIUhI1yFvatLQ0iGcpBhKoKQLipez69etl9o1YuXKlsqdNTcx8Pfvss0btyS7ytuiVyWAoQEJOImKzE5FdbFubKeQXZiI2+xqyimyrXzX1PWO9JEACVSdABaLq7FiyFgmYuuitbhsEWRYlyoM+zJ8/XzHQ8vHx0Z9mnARqjIAosatWrTKyUZDGxGaiJjZ/k/b0QTZDFG9RthQOnHsHD/76PPpteRXLUuLvqGtJqYcx5+jHinvVN0/tQPId1Xbnhc/Er0S/LROxMeX6nVfGGkiABEjgFgSoQNwCDi9ZLgFTF72DBw+uFmHFAFuWiuiXRrVu3Rqvvvqq5umhWhpiJSRQQQKiKIiNgthG6F2urlmzBt7e3pAlT9UZMjIyEBQUpFX5888/m8WlrNbgHUYWH52Cdhum42xBeRVdxQ+RJ5Hr+wiiHlmKZwMalZepQueS0/ZgyO5Z+DHpnOJe9dTVbzDp9xMVKstMJEACJGDtBJytvQOU3/4IyFrtWbNmaR3fsWOH8kZWO1HFiCgP8mZXH+QBbc+ePahbt67+NOMkYHYCqm2E3qA6KytLmZ0YNWoUli9fXi0yyT1/4MABzah69+7dyp4qv//+e7XUX7lKinEobhVmnN0OeATib23/hN4+4hAjG7tOfYxPEuOQ7+iJp9u+hmGBwYg4NwOL4i8hv9ARL+16BW1D/4SPm7fUmvzP8Y+xXdxHp/+Gx/dcxYddX0cL51xsvPAt/h13Do4OThgc8gf8sVkXANexYN8M5Ia8BO+477EpsyUW938KgaW1RaUcQnpRESa3fxuvNKyP3LxkJBs8S6/mIuLcN1hwJRJJAJ5s+SrGNAoruVaYrrUHNMaHd/0J7b09gNwLWHD6/7A1LRl+LkF4tNVEPB4YhNTs45h2+Ct0bvY8EhL/gyNpLhjY7FVMCS1R8hJTf8Gbx9bjKoIxOPjGnh5apxkhARIggRogwBmIGoDKKmuOQH5+vvI2Vm1B1orLvg93EqRO8X3fvXt3ozet4r1L3sZSebgTuixbnQTE5ar4s586dapRtStWrFBmzkzdVxtlqkQiJCSkjEvZY8eOVaKG6smak7MPbxxdAoNTA7R1LMK357YjB8CyE+/hmQun4FOnORo55+Nv+6ZhUXIaPD1CEOTqBgcHdzT3aY5m7sZ7wQR6NYQ8Yru41EWbOo3h7piH5adm4ZXI3ajn3Rwt3Nzw+Yl3MOVcFPJQhLiMaHx1aBr+m+uKtr6BcNF1K8i7OTwcHPDD2c+x8tJ+JDl4oVFpe7+c/QfGnN4BeDdHW886mHvkbXx4LUUpvTaytD2PxmjrmYbv4o8jN+8int/9Nv6TcA0tfZrDKf8S/rJ3ItakpKCoOAtR6dFYEDEHl4qC4W24in9HTMRv6bnIzY3C8we/xrE8Z7TyAVZFb9RJyCgJkAAJ1BwBzkDUHFvWXAMEsrOzIW9E1VAd/vHd3NzU6rSj7BtC178aDkYsjMC8efMgH/0GUCLiwIEDFUmrY9MscSkrG75t3y7v7IEuXbpU28aPSoUV+FOUE4PrRUCA/114r80D8FD+Yx3F8uhT6N7uc3zRohHycmMxdtcUbIz6DU/2eBaPJJ7AmTw3TO34Glrpn/gBPBQ2EptO/YIjfn3w9w5PANnb8fnFSISFvY8l7TuhqCgbL+96Dvsur0Rys2dKJeyFL+75G1qZyBtabyjWtI/HmJNr8JdjBwEE4h8Dv8HjXr/jP5EH0bS0Tik2eftY/O/0erwQ1A0rLkSiKOhlLOn5oFbjhbgvsT8rHWO6LcSbjeohLeU3dN35T2xNPIt7GpZkuzv0JSwKvxvx19ag//6vEJkdj5DMrbiQk40x3d5Uyp2I/RYjjvyk1csICZAACdQUAc5A1BRZ1lvtBGTTOb1ry2nTpt1RG2+88UaZBzCpc9OmTVQe7ogsC5uLQG5urpHnJLVd8RIm9/edhpdfftmoCnMr1Z5+I/Fm6x5Iu/Ax7v7lWcyNOoGCzKtIALD/90kIWzsM7X6ZjIjsXBQU56DYYCTu7RM513AJQHOvYCWvk4MjvJ29kVcUixzVkVHQXWWUB7XiJs0nYuOgT7Gg/Ug0dUnE3H1/x5X0RFwBcDbqLUU+kfGXtEwUGbJQmFlyrZV3SXtqPbk5V5EFoJFHPeWUr2uJo4b4vAw1S7lH03LlZuJJEiABEqgBApyBqAGorLJmCOh3rZZZA9ON3Sraqrh/FSNp003sxJPT3LlzK1oN85FArROQ74F4Tho0aBAefvhhXLx4UZFp//79kE9AQADGjRuHhg1LX2NXUmLZd0IUaqm7qKhIcWEsMx93qrxXWAxDAR5o/lcMa56Gp3ZNxQ+XVmBCyGNoAKBBy3/gh9at4eIAZBekI8fBC14OpTUbCkp3jL/NOzLPxmjhAJxJvYBcQxCcizKRnJ8BD+f28LrNf8ecgjTkO3jC170JHmk2BtuubseO3GTkuQcjxAFwavJXrO94N9wdHZFbkI40gzsCiw7B1QE4mybtdYK7IQ+pRYCnVwjq4BCiMq6g2L8REnNERQKaeIi9R7nW4Mp1D6/G8MYhXFDKBSGrILsUAA8kQAIkULMEbvMTWbONs3YSqAwBvVtV8cJUlT0fxP1rnz59kJ6erjXt6emJ77//vkbcYmqNMEICNUhAvgsxMTHKrINeMX799dcVhwOy7C88PLxKEohy0rhxY005kTqvXLlS7sxHlRq4RaHs9LWYEHEMQR4euJqbjYZ+reDp3BaDGrXA3PPv4ZnUNvB2BOIzoxAc+hY+b9EKDTyDkF+0F+8emoUOIU/jrSahN2/Boxv+0CgE71z6DH/I3QqfoljsyXDC8z3GIdDp5sXkys7zC/BhQi5aengDhSnYn5KIjmGT0dC5KYY3Dcf06I/xh6wtqOvijKTMC8gNnoL17Urbi/0Bf9h/CnWLU1DoMwyftuyP5p6bsPr0bMTFByMx8wyAIDzWoB1giLipIPV8eqKh689Yp5Tzx5lk89up3FQ4XiABErBpArd5PWPTfWfnrIRAfHy84q5SFVfcqorRc2VCYWGh4u6yQ4cORsqDbAwnnmxqwqd+ZeRjXhKoDgKyZ4PYP8h9rQZRluW+l++MLHmqShDlRF/nunXrzLITu6fv43jMLxDphXnoXv9hLOz6FLwd3PFCx7cwt2UveBbnIr24CL3a/BUftCixUhgc8hieCGgHJ+TCz6WOSXdd0SIgHJ3rqL6U3DC+4xy81/xeOBdlIdehPmb3/iemBteHE1wQ5h+Onj7le2DrGzYKD9QNUVy4phtc8c5df8e37brD3dENj7V9E5+1fwD1kadc79BiEha36wBA2puJ95qHK+2lezbF6816wdWtGf57z4cYXS8U2YVZaODfBz/e/xX6eHnC2ckHnQLC0dbLBzLB4uZaDz0DwtHQ1R11PMPxXdc/oW+dOshyCcC8rn9VrtVzMTH+MKHAJAmQAAncMQGDjYbhw4fLaljlM2HCBBvtpXV2691339XGpl+/frftxOLFi7X8MqaSrmwIDw83qkPq2bFjh+Hq1auVrYr5ScDiCch9Lfe3+huoHtu2bWv47LPPqiS//nsr9YWFhVW4HrV9OW7btq3C5ZixegnY2zjo+1u9JG2rNmvmtGDBAu13LjQ01LYG5ja90T8b1UbfOQNxxyoYK6hJAuIJRm/rMHnyZDz99NMVblJsHWR2QZYu6cOiRYuUPR+Cg42NGfV5GCcBayUg97XsaSL3+bBhw7RunD59GvIdWrJkCSrrwWzGjBmQnanVEBUVZeRSWT3PIwmQAAmQgO0ToA2E7Y+xVfdQNo3Th08//VSfvGVcHpL0yodklt18q3vn3lsKwYskUIsEJk6cCPno3bGKOOr3orLuXkUhd3V1xbfffqv0StzGRkdHo2nTprXYSzZNAiRAAiRgbgKcgTA3cbZXYQLyoB8RccOAcO/evRUqK29WxTON+pCkFpK3p1QeVBo82hMBdRdrvRtk6b/sIyGelioTxKuTPjRr1kyfZJwEbJZAetZ5/N+ZZZh1chHmn92F5FruaUzSenRdNwyzLpd4X6tlcdi8nRGgAmFnA24t3U1KSsJ9992nidukSROIb/vbBSnXsmVL6D02+fr6Koal8vaUgQTslYDs2n79+vUy3pNWrlypKBK7du2qEBrxynTq1CnoN2AUhZ2BBGyZQH7+VTy5ezo+OL8ev1zdh5UXPkGfw/ttucvsGwnckgCXMN0SDy/WFgFTP/PHjx+/rShi63DkyJEy+fbt21fmHE+QgL0SkId9WXL08ccfY9u2bRoGsZkYPnw4Vq9erZ27WUTcxopCr+5SLQr7Rx99dLPsPE8CVk8gLXM7LufmYViHeZjdtCnycq7ieL5bab+KEHVlHRZfOYkkWSrbYDjGNhGvWwCKC3Dwymp8fVVc8wbi2dZPooevF5CfgNUxq7AxNREeTj4YGPI4hgY2ApCEeUc+h2vwMwjIXomd1/PRJXgUXgwJUarLyD6BBSfXIA7+6Fk/oKQN/iWBWiDAGYhagM4mb00gNTVVW2MtOcU15e12wBXD6jVr1uDy5cta5apLyzZt2mjnGCEBEoDiWODXX39VbIJkhk4N8h3y9vau0FI/WRbVvn17tahSTlwuM5CALRJw8whBHUdH/Bb9LX5NjES6sz96+ZW4+I1K/BZPHP4/HMrLQnruNcw79ne8fa1kZ7BCfgAAIABJREFUr6GzsR/h6aPLEJOTivTck/giehOAPMw98h7+cnY7EvKzcCH1GKbsnYTFCSkAcnEwYT8WHnoJ31y5jNiMCMw99hKWXhcXzHn4+5EF+D7hPJLzo/HN798jTXwQMZBALRCgAlEL0NnkrQnIm1B9mD59uj5pFBcvTZ07dzZSOCTDsmXLcKtyRpUwQQJ2SkCUgEOHDhn1XvZFkeWDsjP77cL777+vZZFyPXr00NLVH7mOBftewdADa5Fa/ZXj/LV1GLp9Kn5MiquB2stWqbT32yvYkioPjTcPqdnHMXHnK3jrfAQKb56NV2qYQB33HvhnmyFwyTyESfvexOM73sLKLGk0Dl8eXAWv4HFY0Wc2lvb5EI/VdcbSIz8gHnH4+uQO5Po9jBV95ijXPmgzAMjaijWJF9E97E0s6zMbS7q9iABnJ2y+skPrRUDdJ/B/98zG552ehq8jEJFyTim3KyUR3cPewA995uBfHUfCW919XSvJCAmYhwAVCPNwZisVJCCuIvUuV48ePXrTkqI8yJILvaG17DItnmUq8vBz04p5gQTsiECLFi2U78zUqVONer1ixQrFNkJmKm4WZNmgvpzMAMr3smZCEeIyonE68zqKaqCBnILrOJ0WjeTC/BqovWyVSnvp0UgrvLVaUFSchaj0aFzMzVI2NipbE8+Yg4CDgzO6t3gBGwd9hL82vRvuBWfwl62v43RmAo4XAnHxS9Bl/TCE/Twa3ySnAQUXkZZVcq2ZTzP4ODsDTp4IcvcD0iNx1QCEB4TD1QHwdw+Aq6MLkvMTtK44O/vD18kZbs5ecHIAig1FWrn2Aa3h7OAMP/cA3GbDdK0+RkigugnQBqK6ibK+OyIwc+ZMrby4XJXZhfKCeI8xDbIUo6KGoKZlmSYBeycwb948yMf0uyWuWiXczOWrlJFlT+p3V5R6sa0Q17EMJGBrBOq4h+GFjtPgUnQF71+ORJKDP5oDKAiZgi2dBxh3N/eYko7OFssIXfAOgVgvXMi8CtRvgKzCbBQaCuHnchubBpNyuhoZJQGzE6ACYXbkbPBmBEztHMpzuZqTk4Nu3bqVqUJcvFbES1OZgjxBAiRgRCA3NxefffYZpkyZYnRevl+ytElsi0zD4MGDNQVCrkm+mykcpmWrmi4oOoc5+/+NVUkXkAageZ2u+Mtdf0HnvF/Ra89XGBn+Lj5s3hXIP4ExW95ETMM3sK1zODYc/wKLYvfiXGEBfJwD8XjL5zGtRe9bivHm9mHY4jgAQQWH8XtmBhrV6YbHgurjv7EbcSXXCS3878GinlMQ4pyDn88txscXt+N8bjbcHb0RHtgDc7tMQYhTBj77fR6+iD6CDNRBe19/XZv52B29HB+fW48judlwdfRGn4ZP4P1Oj/ENs45SbUZjL36IFy4XordffaAgEVvjziO4/nNo6x6MJ1p2xORzX+L5wpMIcfdAcsZprHMdjahunTEtNBQvXPwvnj+WhhBDIi4WNcFXd43A3b6rsen0HLyeGY7sjCNILPDG5BBZvpt3827W6YP2Hsux5/QcvJkZhtPxm2kDcXNavFLDBLiEqYYBAwbkF6QgNjsJOcXFt2ytsCgLV7KvIdVMU+i3FMbMF2XTt7Q0eQwo8U3/448/lpFAvMd4enpCdtNVQ2hoqFKOyoNKhEcSuDMC4p5VvmviqlW+X2rYv38/5syZo8xSxMUZ2wnI90++v15eXmp2iLvXwtssz9EyVzpSiNUH/4z/SynEB/evQ9QD89Gj+BgmnfgVfr7t0dWhGLsSzqAQBsRc3YKDhXUwvnEbxMdvxl9idqNT67cRNWQ5Pmzgg8UXVuBKvhio3jokp+7HqC4LsbXHCKRnHMSXV+PwUf/l+Ferbjif8iv2pScgMfMgPjz/P3j4DUfE0HX4pmU/HLr2KxZfPYHErKP44dIRtA1+GaeGfoSezjfeSqdlXcB7kSuQ5zccR4euw0/t+2Fr7GJsvU6j9FuPivmuBjd+And7O2Lz1X3YnBSF4a1ew+5eI1DPyQ39W7+Df7e9B5kpEcr167598Fu3ngDcMLDDP/Bes6Y4k7gPm3NyMKbZcAB18FGvOXjC3wf7r+1DdHE9fNL3CzxVLxCAEwI96qOBmwdknt3R0Q0NPOrD39kVQAPM7/pndHfNxK70K5gc/iY6+zSEnxMXMpnvTmBLKgHOQKgkyjmeT4zAguh15ay3bYm/9RiNJuWUKXuqABtOzcbUy9fwQb9/YrRPvbJZSs9Exq/AE4f/i6Ft38Hclt1vmq+iF5ITVuFvMae07A6ejfFMw5HoUddbO2cpEXEpqQYfH58ym1uNHz8epvs4zJ8/v8xbUrUOHkmABO6MgLhqjYmJwRtvvKEoDmptr7/+OmbNmoXdu3cjPDxcPQ31e6vuUr1582bExsbWzC7VhZfxUwLg4mbA6pPvQxzPXi5yAZJ+w3HDQ/hj40BMvLIXO3NH4NLVaHh4NMZQH29EXjgri7FwPn4tJiU7ITcnHcjLxcasHPTVenKTSJ2RGOEfgOKsXghy+S/qBw1GB1c3uHiL603gemE2rmfuRXwRMK754/B2BLo27AKc+RlRmcm4bthfeq0/3B3d8Ejow/gm+SelbGrWfiQUAP65kZh26H0UFJRsUXY4/QoGedxEHp42KwFnp+Z4q/MbeKu8Vh3dMLDlyxjYspyLjp4Y32EOxpd6ddVyuDXGe3fP0pI3Ig3w735fa0lPny5YM+BG2j+gN5Y8cGPGbFDDG3GtECMkYAYCnIG4BeQiQyEyCsQt21Vsjt+P/WlXkS7pglzcei5BX6kTOgS2Rs/AcIS5uOsvlIk38G6GXvXC0b6OvIW48xCVtA2b448iOj9DkTsuYQue3PVHzLh4BHm36cDKE68jbO3LOHyL2dQ7l7CkBrFz0BtLixtXNYiHGFmTrVceWrdujVdffZXKgwqJRxKoQQKqO2T9bER6ejo6dOiAe+65B7LkSQ0yk6hfYii7VMvO8NUeDAblN7jYkF/6m5wFX6+W6BnQBUGuzugZ9iiCDNH4v6gjWJuViGY+nVHf2RWGUjPk7MJspVy+cxB6BnREO095u1ux4OjgBseyJlhKYYOhWGnBESX/Wh2Ud8hQZDW9pm9NlnuJN868Urly4I6eAeHoWKfETag+L+MkQALmJlCAK2knsC8lDgVVbDozLx77kk7gcl52FWuwvGKcgbjFmLSu3w3/qd8NyNiLsG0foH2jp/Bdu5LdkC8mbcZjp9ZhfNMhWBy9BaPDX8Fw70wsOP0tDqZlA14N8beWL6O3nwcScnKQXiBGUgZIuSmn1mFMyzHYHrsMF7MDMKHNn/BEcF1kFhQgrSALucVFyMo7g9cPfoKw0MnIu/4t9qQA/UJfwuvNSuY9khN+w5u//xdXdPK/3PkDDPYznV2oiz/1nImhLu7Izb+Cd/bNwNLfv8TAenNxr5czDl9ejYWX9+J6gSM6NhiPD1p1x8kLH2LBlWjZAQev73kFgQ0nYXnrhlh3bimWXI1EnqMn+jW6IYtOhEpHRUEoKrrhU+XFF1/U6pBrpi5dxVhzz549qFuX/1g1UIyQgBkIyIaM586dQ9++N97VyyxE165d8corr2DSpEmKFF999RW6dOmiSdS7dw28IXUJQE9P4ES+F55rNw0D/QMQl7gFf070QWMARV590c77O+yM+RQOxZkY1bIfPB2Bhp4NZJEkugUNxftt7gVyL2DBhZ1o5+iCG/5vNNErHalbtxeCHffgl5jtGF/3fuyPL9nd+y6/xvD2bAU/7MGOmO2YULcz/nd5s1a/l2eY4qoz0KszFt01Fl7FKVh57hv4uvoB4DImDRQjJFCTBPLj8cqRj3Ah98ZDvp97f/yrVyd8sOtNbDT0xuahbypG85UVIzpxE8Yf+QmTu87HtMblTVVVtsbaz88ZiCqOQV5hKiLTovGX339EqG9TBDpnYMHembiQWw9t6zREXvxeTI74EuJ4LyM3HqfTLyOjuAhquTcivgGcQ1CYdxrvn/wAMXmF2rVr+XkoNuTiQno0Po2Yid/z68HPIRFfnPgjfryehey8s5h49F+Icm6Itl7uuJgejUznJvBzvvU6SHfXRng+pA2KCtJxIjcLZ+PX4+UzEfB3bY5Wbu5YG/keXjp/Du7ujdHQVWZLXNG0TnO09nDHLyfnYX5sApp6N0f9onRFli/ir1aR3o1i4jte/wbz888/R35+vvJms3v37kbXJkyYgIyMDCoPN/AxRgJmIxAcHKwo9IsWLcKwYcO0dsUmafLkyZDZB5ltkBlFvaF1UtKNtf5aoTuO+OJPvd9Af68cTN35NMLWDsOgYz/CW/ndApwcvdDPrxkKi9JR4NgOjwQ3UuYCOjd8GH8MaosdUfOVMmFbp2NbLuB8symFSspZ37sbxofei9hrX6Dj+mF4+dwR3B08BmOCWiDYewBGBrfB8biFaLfxZUQU1tdqD/TqhleaDUBGyhp0/nkEwja9gE9SCuAlrj+tPBw4cECZRZaZZLlHMjMzrbxHFL+yBKxl3HMKMxCZdg4JhU5o69tc+bSsEwBn1MfjrQbiseZ348a3trIUbDC/wUbD8OHDZUZY+UyYMOHOepm+x9B8zVDD+FN7tXrOXF1haLtmqOHVyD3aOX1k+9HRhuYb/2a4ZCgwbDrxN0Pz9c8ZfslIMajlZkSfUrJHnJtV7rX0nGOGB38eanj06G+GXIPBEJe42dB+7VCDlIu/ttjQac1zhnVZUkW84U+bhhqG/H5G37wS33/qT4bmayYa1uXnaNfiYxcZmq8ZaZh15bJ2TiL5BRmGxzbdaO+n49MMzde8ZDgkjZsEvSwmlyqUfPfdd7WxUcdIjikpKUp5/Tk1rl6rUAPMRAIkUOME+vXrV+73WG141apV5V7ftm2bmoVHMxNQf0/laK5x0Lepxl999VWz9FxtT44MNydQk5z0z2I1Me4LFizQfmdCQ0Nv3skKXMnOOmt4YONjhlGHV5rkTjS8tXmoofmWLwxJhnTDl9uGGu7eu8Rw6OL7hk4/DzX02DzXcCUnr7TMZcM7ByYpz4zNf3nO8NmlBOX88ctLlHNzL581qbvqycWLF1db36siBWcg7lApbFZHNaXOwqqT76HfpuHKm61nLilbVN5h7eUXr+PbA6Fu1/FJxMeYdfTf2J5bBw83UOUov4x6NqMgA4Az/FydkZB+EC9vm4Dwn4ehzYaxOHZjKbOavfRoQOTVlRi3ZQxarRuGe/YsRI7cttUYZLbhvffeK+ODftq0adi0aRNMXbxWY9OsigRIoAoExM3yggUL4O+vd0da4kVt5MiRio2E6bUqNMMiNkhg4cKFym+9ePtisF0C4oRhzZo1WgdtadzjE37Cc5f8Mbn5/SjO3o5xZw4CyMCi3yYjwbEP3mr3NAY5FmPBqXdxMNs2Z92oQGi39h1Grm/BtAsH4dVwOqKGrcLHDT3vsMKbF8/Jj0JCHuBdFIHNSXEY12E6XvC9vauO/IIkLL58Gk5uQXjAywc7L67A/zKu4+1e3yPyoa8R7mbaZiGKxfVscR4+Orcep9AQmwatxLZek+F+EyNC0xoqmh41apTyMKLPL7tJz507V3EHqT/POAmQgGUQkAfA69evG7l7FclWrlwJ2eFadqWmEmEZY2WJUqgPlMuWLUNCQnVYoVhiLymTKQEZ98aNGyte3kyvWUL6UOzikiWOa4eh74mj5Yvk1h3r7vkjXmzzKp6tB1xOOoVk1MHz/Vfhg/YP4oGG9+Dxho1QWJSP1OJb7zZffgOWf9b6F1haCmPvxmgHIDHpJ0zavx6HEnPEhKBGQkFmLDLgDifXZmjnCiRcP46IwLbo5u1STnsp+OrwP7De0RHpOedxOK0IoztMRjN3T1zzbAAnRGL5mfnYbIhHZB7QvrSGwNJrsw+/h9CGo9HaxRvbM+Mw4+iHyMw+j9xqnoEQ15BqkL0evv/+e4wYMUI9xSMJkIAFE5A3jfKdffLJJ42kFCcI4uo1JSXF6DwT9klg1apVeOGFF5CYmGgEYOzYsRAbm549e2L1anHKa00hC6uPL8BGQxfM6TQEYvZenSEh/RDmnNmIjo3H4+kGzaqz6lqt68qVKxAvbbJz/Zdffon69S3HuqBFwABMbV7i/MHDS5hXVAHIwS8n/46/x55CbH5pGaegWuVck41zBqIidJ18FJd6bT19tNweroHoXi8cjV1LX9u7dsMXd/8ZHdxdke7gj391ehI9/ZvBDQ7w92qGngGt4e/kDLVcqHvJDIWXR2i518QIsGPdcLT39lUcArq6+KN7QDiUcu5NEeBUjKw8cSmbhcPJ6zDq18nYZ7IEqVnd3ugZ0BKeReIFKgvOfj3xQ99PMatZF7g6AH1Dn8X0pl3hbshDkfcAjG7cR2uvb6NH8VRgJ7g75KGuWyCmdP0LJgQ1Q0FRNro2GIK+gZ1KZNGIVE9EXEWKYTWVh+rhyVpIwFwEZK8WcUeqd+MqbZ88edJIBNml2vScUQYmbJaA/K7LTIPcJ/369UOjRiV7aEiH4+PjleUuYmwty+NkDxHrCAU4n7QfmxOjb7WHdJW7kp0fj9/i9+NUVslGq1WuyAIKquOuF0WWOAUFBZXZ+0mfx9xxP49QDGrQS/nc61MJlTDrNCZfiIBj3RGIGrYaX7W84Y3O3H0wR3ucgagIZc/2WNpntlHOJnX749u7+xuda1jvPiyqd592rndISbR+s+ewVH1x4GFcLqzRU1iq/oaaXPtHrxttBvh2w+K7uykVyg6ssc6t8WO32ejqCXx+6HnMjYtHhjgo1m01ERg8FkuDx2rylIk4++HZjjPxbJkLUk8zvN37faMr07vdkAcYZXStOhI7duxAy5a24d6sOniwDhKwRgI7d+7EmTNnjFy5mvbjrbfessI3zaa9YPpOCIiSIG+hd+3ahTFjxhhVJUpmw4YNFW98y5cvN7rGhHUTkHE/duxYmd8HWfYoHtxkNYK8jLDK4B6IgQB2J2/G0O0HcS0rDoDtupznDIQV3qXtg4egmWMsntgyTFmn92mqC+5v8SburmOFnSkVWVxDynIHmcZmIAESsF4CHh4eyoPA4cOHIa6XywsDBgwo7zTPmZGAPKTL2/6a/tyqSzIDIbZu8mZa7hUvLy8te1xcHFasWKHIZy1uQEuET8ZPJ+ei79qS/88P/joDuzLSkZL2G3r+/AieidhUkq0oFlM3DUPHvcuRhjwcj1uH5zePUf6nd93wHOZfOIbM22z4qsGqxkh13g+yRKm8IIqCjLnYQjz44INaloiICGUZZL169RSXv9oFM0WcnH3wQMP70C8g1KRFN3RpMPD/2TsP8KiKro//k83upvcOgSTUhIQO0oN0aRYQXwEVFBQrFlDwQ6oithcQX6oIqCCKSAm9SG+hQ2iBQEhIg5De237P2eTe3E0CpG87w7PcmblzZ8785mb3njsz5+AFj6ZQQo4m7r3xQr1WEHa6NnMpPifzwvK+CzC2fnv42fthTss38UL9LvA0U8DBsjFe8OqNFpKVLKUa0b9kVUw36cM1UtNh1Tbjqg8d1iMZpWZcyRQkBybABAyXgNTMIpmLrCvzoYZLtOo9k5rr1Ea8IpKXvl+kclbWDKj02oq0XbUySarv9g9W+e5dpIpTqVTHLr2u8t0yUrU8VaVSZUUUnTv2p0qVe081bvdLqg6H/6tuJiF+o8p3y3OqSeEXVfeTTqg6bBssmg89fmW2ynfbKNXupIeqOw+CVW22DlZNunm+auJV4Copp9qMP0oUMr9aXrsVeT6Q3i/VNeP6KPl0NZ/NuOqfzsUS1yCBQ4cOabwBI6suwqcGm+GqmAATqGMCly9fVv8t0wZraaA33ytXrpRm6Uw8J+8MhuwYgl7H/gEZvK5IuBi1TP3W+PvosIoUr3aZNZemFFmI2f4Frla7Nt2rgL7/6e00mQguHQSrTVRGJ0NBPP6JzYK50hZxd1dgzq09uC2zARKO45jKDeM8PfEw5Tp2ZQPnHoZBJrNCb0cvRKWeR3oBYJIfgzmhK7CNnO0VZOJ4qnF4IafviHPnzpUZc+H5QGfHWydvwroTivdA1B1rbqkCBOgHQgjSOE1t2toWbWKn6U1ra2uhGB8rSaD0A10lL9f54mTFS5cseug8sGoImJmZqWF+k/y20FrmJ4Xg4GC88cYbTypW5+dlpk5obOuKBBtXlGfTjgS6Gj4LQ65EY0nf5ehnAbhYOKOehSs8FMKChtoU+zw2R1yBvP4MXG/bvtoN0YO6rgbhRRLtj/j33381LDfRbwN9/vjjD9ByOJ35e1cVIAMqkMn0vbEni9FaoJ5FAzgpzNDcdyAa3F6IhbfD0SElHDaKFmhh7YDYpBy119uwpLOISSlaWV7Pwh31io2t1OUYkVUkbVjCatOmjXpfRGpqqtoqE+2PEYIw3nfu3IG3t7eQzUctE2AFQssDwM1XjECrVq3EgvRF06BB0Q51bXzRiYLoSaS0RSupYx896UKlxBTMQQoXffDBB+qHDCHNx+oR+PHHH9UPdFRLbGwsQkJCKl3h1KlTK32N+oKCdByJ2oy19yMgM/XCR/4vobGlOZAThdW3t+JkWhJszJzxjM+L6OXghNSsG/j2ygb41R+Bh0mbcTVNjiCvV/Gyh5PaNOPp6G1YGR0KmLvgw8aj0UiphKuFD+wsbdTW7+5nXMa8a9uQaWqOF7xfgV/+CXwVeQtAFpZe+BL/OA3CVEdH+Nv5wMlMUW57neu9gtfqOavFT0w4hv/ePoAESe/fCZiEltQHSbj74AC+vnNMndPcrhNe9Q2CZX40Zl79EzSfY568GZ/fzMHcJl0lVxlmlHxE0H126tQpPP/88xqd1Dnzr2a2aG8px8ECR0xrNwv9HJ2RlXYJX8VnozlJbtkNQQ6/4LeIH3BfFYfGvp/AwwzIsWoIpQnQ1fNlLArsB+Tex+bIPfC2cqZbzajC9OnTQS8i9uzZU2a8BbOv9P1BJn85aJcAKxB1zD8nLwmXUu7BxdoX3uYlG8YqKkZiZiQi8q3QytYJsidelIuI5BuIM/VCp8qYIntivTVbgDZQjxs3DoMHD0Y6Td0CuH79OuLj48tt6Pz586APBdrwJYQDBw4IUbRv396oZynI0gXxJJO4xhYEc5BCvwWFiZZE9OnTR+2XQDjHx/IJ0N8fcaRAVnK++OKL8gs+IZd+5M3NzUFLEaTh66+/rtJbzt03v8EHYdfQxM4H1ojDsujmmOvthY+O/x8OZpuiha0HolLCsCXmX/zUfRXayRJxLP4U/oi7BD87X+RmR+Lf+yfh1fM3NMk/jI/Or4TCshncs+OxPCIEXzeth1P3TyFT9RQKCiIw/ejnOGvigyaWcnxx4lcsbNcA6QUF6uXamfkZSC3IR2p2JPbGnULrBqPRQV62vb1x/6KR3Ua0lt3GK2f/i2S5DxqapeF8cgxsbQKQrzEJUID4+N8w/NRmWNv7wcMkG7/dWIhDmSZY0dwHafnZoNZRmIn0AjK7ZxzBw8NDbdqbZkx69uyJW7duqS04Ue+Fv3f6LaDfAHJgSE7KtBOs8Ua7d3D0xHJMOjoW9O1ramaNpl7jisUxRxfX1vgt6QiSTerhNe/moIewxo7dMcrlBDZGLkGjO4sAEzO427XEf901bg7tdEkLrdIsMr34ovGmGSipJS76PqcPvVQ8eVKY5dGCkNyk+t5lDHVI4Hr8Frx6biM+67QKYyqjQOQnITgmG275wfgkrR1+9riFQten4PdY2U1wLGIFpsfYIrjPl2qnc48trqWT48ePR2RkJLZt2yZKQGnycEuBvNnOmDFDPPeoCK2tFkLTpk1B1mAoTJgwQf0Rzhn6kayakPUSDpoEPvroI/UyOHqLRWYEOZQQoAcv4iME+vurivO3WbNmYejQoUI18Pf3h0Kh0FD0xZOVjtzG3+EXkO/0Cv7sNAwWqiwkFJggJnEjjqUlYWjgAszxboQHD3ai68ml2PXgKtoVG3VrVW8MVrcagLi4v/DM2Q24nh4Hz8LbSC0Eunm9jB99/JCsVgxiS6TKuYsbOUCeR1csDnwBmflJcLF2xdcZV9RLmD5u/7V6CdPle2V3Imi0d2atur36OI57ObaY1WUenrOMxKt7JyLb60O0tZLMPuQnYN7lPSiw7YC/u/4fnExysOHcO5gStRGX/Rbjf0+9hue3Tkdhg4/xY1NtPSSXINJGTDD/SstaBw0apCGCYP6VHi537Nihca72EjZ4tcNCDIRNkcFOi05Y3K0pYrOSQa6ZTGUWqG9Z4kysu884BLsPB0ws0bjoJwowc8bkDjMxKisByQX5agXC0cIDrnI58s27448e/rA296i9LuhwzTQDNWXKlDJmX2n827Zti5QU/fePocP4Hysaz0A8Fk9Nn8zCzog9yLV4HoNci6a0K9pCXMZ1bI5YjTzIkISH+DQpET0LveDn7vmYKuRo49oZish1OJIYC3933f0CojWuZBueHjYo0BIlYZkSmXyjaU0KtOb6nXfeUcfpvzVr1ohxaSQsrGRD49tvvw36UGjWrJn6y0goO2bMGCFqEMfZs2eXUR7kcjlGjhwp9o9MIhp6oHuE7pXc3Fz1Ommhv7S+ln546O2WsS1/I6WA/s4oEJt3331XwFKpY9euXdVveYWLvv/+e9C+pEeFBQsWaJwis41VYp8ZhVsFQFP7ZrCS0fyrNVzNgKvpd5BKf9t2jSAzAdyt6GGtEPey6MGiaJbXSm4PC1NTWMgsAJUK+SiEp/OzGOF5GX9dm4mg240wqc1nGCg12W4ZhE+a7MHc8D8QdH87hjf5DJ80rZi3XI321IulCmFj3RSOsq3YcmcbCixuIzTXEq85lzxYqiHlp+Fsfh5cbFzhpJ5iVqKFQwMg5gwiMzX9/GhANbIEmX+lD72hpu/wv//+W5xtJfOv9JHOTtcuHhlcbXwhvTMsFI7wVUhvphIJ6Jx/eedklvC0boDSv+gKMzvOs05HAAAgAElEQVQ0s7MrqcAIY4LZ13Xr1uGff/4R91ldu3bNCGnoTpdZgajTsbiFc4lp8GvSDS6VbJemM1/zDMBnV/cgyyQWNg1G4T+PVR6KGvCy8oXc1BTH71/AWzqsQBw7dgxKpVL9g/A4NDS1KX0AlsYr8oNBDq7Gjh0rNiGN6/KGQlHgx0Toh7S0QkUzNzNnznzMVYZ5avHixWLH6EentOMiYVmTWMgAI8nJyXBwcKiRnlVn86J0ZoOEobeJ1Qlh6bS0qmRPlLmFh1pNiM6inQXOSMkldcIU7kpyjPNoQ/rmCjdMa78IH2XfQt+DU/H1jfUY2FnzjfZgv68w2A/4/vxHWHJ9Crq4rUdVX8PkFDxEZkEBIuJX4tMsYHSLOZhY2smtzALNTGW4mFu0lJM4RWfQrIgHPIS31dWBZ4DX0m8AfUhRLX2vGWB3jbpL9CKsS5cuogKhHRhpWHFwJOaZ9EV40Du4dGsmnr8Rh6VB36Ovdek/6PIlzMq5iNGHpkHlOgl/tA6CsvxiOp/LCkQdDlFy9GGchRJjnYqmnrNy4jH60DikWXZFftYZ3M0qhK99Z/S2BzZEH0ZyniWCvIZjfssXUZgRhg0PbfB+kyAsyg5EF1zGtqSHGO+Qjw/2jMMJ8+7wLryIc6mpcLdui2+fmoWuVoCNuSM8TWSIzJRMzddhnyvbVKdOnaq8rlFQAO7du6d+IyW0XdEfFUEB8fPzQ79+/dSX0zIoWq+tb6FDhw4gj8CklHGA2rEZ3R9k+eeXX35RI7G3twc9ZOtzyMrKgnRDcnh4uMZSwMr0jfY5ODoWvTWl+5+WH1UnkBlXmq2osWDZAaPcPfB13Bq8cyESrgXRuG/eCz80eho+5vuw8coPyE70RVTiKZiZuuNZd1rgeeWRzd+OXYXZd5PQyNIE8oICNLQuZd0l8xBGnNoFf2df3E0lo66N0cDSCmZW9WCFM1h98Uec8uiH5568GU0tQ0bKbaTCBZ08OoPc6FnlxiAsuzWaSlYwQe6GST5+GHgtBC+dW4EA03hsioxDW+/30I0ViHLHkqw10d90WlpFDe+WWw1n6jgBGmfaS6Vby09VuJ+dDxQWIKNQvTupfIrppzEn4jr6NH4Fnc2BfFUGMvMBeV7mY15xlF+VLuWyAlGHoxGWcgOAA1o7aDoAj8xKxIpuv6IwejHGXz2EdSb9sav/n1gX8imWxB9EZM5ABNq2waKn2uBI+DJ4ZiswptWkYi+IRRuNE1PO4eWO87FaeRI9j/yCn6JC0aV5AEyU9uhoYoq1WQ/rsKcVbyooKAg0q7Bz5071RWRpg8y10hR0VU210gY6+rIRgjQuNQFHSzoEpUMoS0eaFpVOjc6bN088TW9ihSCtS8jTlSM9/LHyUHY0hgwZIioQZc/qZk5hYaF6j5AgHVkneeutt4RkpY607MPMrOhrn5YeDR8+vFLXV7QwbWxt2bJlRYtXsJwlxrWdg9yLX2L9g5OQ2zbG/AbtYWFuiS3dv8KHZ3/GwbiTcLBqjF+fnoqnrICkDCXcLFzhIleCzC3IZJaoZ+kKG5kZ6tsORePwL7EnPhW+Dr0xw/95mOAOXC1ckaWwgIllN4x3PoV5cSdhrfDAb72+RBOy7er0PF5xuYLgtItQOPaBwsZabcaV6jQ1Kac9i6L2rGwC4Cw7ilMxJ6EwAVJzE/G/m9uwfcBiNC9auQnADM2azMB6xUZ8ErYD9Nqnb7NJ+Lxxt+Lve4VavkKzCmotFSSrb8Vof9zVq1fRvXv3MqLTrBt9/9Hmf+GlUJlCnKE3BOiF4PLly9Vme6VC0+8bbaan5WvlvyTMR0LWQxSYOcFKlYrk/HwoZTZwUZImXoCErASYyp1QmJeEXJjDzcIGKMxGUnYqsqGC3NQcTko7mBXbacnKS0JiXi5kMunGdjN0bfoZDvkWwMm8eLa3MAf3c1KRq1Kp23OUy/Ag5SxW3z4MC9d+8IIFPJXtsbrnzzAxs4P6/YEqH2m5KUgpyIcJTGAlt4O9nF7+FclJfTBXpSAtv0DSBykN7cRZgdAOd41W/V0GoaOFJe7btYDC5BCGefWCp6kCLWk67GEc4grzEFh8RfdGb6HsVyYtBx6KsW4eUOT3RQPlL8jJfohcAEoo4W5RYqlIo2EdSdBmNzK1uWjRIrVE9CaJvhRqY3+C1AfC9u3bkZdXZMmETFNKrTg9Cg1twBUC2csWAln4kZ4T8vnIBKpCgBRpYeaL7tGqbgh9//33NUzY0swCKey1GUjJphmwWglmbnin3SKU7IIqbsWiCRZ0+6ZMkw5WbbG+V4nTOk/nvjjcp69Yblq3+Zgmpijig+U9S8r3DfwUfYUvX6Gc3AmTO8/HZCENfxz2GFacql+2vb5F7f15+iPEmPljU/c5aGkBfB8yHkviopCeD0BUIIqq6dBwGA43FOoUGwLQAsv6lsgnPWMscdq/RLNbt2/fLtPlTZs2ITAwEI0aNSpzjjP0jwCNNb0woZlWaaBlkLSB/vHWth7g88NvIsSuH+qlHcf1rAw4mPviw7azMNo5G58fHoerFq2Qn3YFhYpB2N53IH49+QP+engTCSqV2hz0Ky0+wScNA4D0EIwK+RGX01NgoXSFJ002qLdXFeBI2A94+27xEibzTMwN+QEbk26rjTI4mLfF122H4atL29XiLzk5Dkvs+uLCU0F47eA0WHoWLWG6GLkO029sx83sTJjAFF7WjfFZhx8wwCZBLWeIvBWcC27gTlaOug9fdl6AAbRCU8uBFQgtD4C0eTOTR7kukpaqbDwHcVkqQMenv+kB/tdffxUtKtDeBNo4RZ/aClILHvRFJYRdu3ZBmHXIz88H7c8oL0jX0Uvjr732mqj80CwKmZTlwARKE0hISEBoaKg6m8wX0+xIVQIZBiDfF0IgqyXStJBfV0dfX1+NpujFACkyxh58HdrDOeEcnt9bNM4WShe0ajAJ/rWrzxkEduFvRWppT+iYk5OT2jQzWWfiYBgE6DuMfHyUDqQY0qxDZULaw4sY2eUnDFHdwLAT3+C368EY0rVIqY9NvosZ3X/Gqw4OOH59NpYmxuPTTj9jvL0Mf5+egM9u7cC4Bk2x79ZGXM5U4LvOa9DF7CReOr6kXBFu3j+ClQlheL7xR/i+eTfsCv8H1taBONh+BBqd2IUpT6/FeBsgLftiyfUF4Vh4dQNi5EHY+8wkOKcdV8s5/+Lv6FwsZ1pWHCZ3WYoBqmMYfHwFVt+7iH5+JfvASiqr2xgrEHXI29WCLG5cRXSmCqgrowr5WYiECg5y3f+VojX70mUP3bp1w9mzZ9WWk+pwmDBgwAD1h9osKChQv+2iOJlHlVp3epRMtJFZ2MxMNvDpAY8C9c/GRgdeGzxKcM6vVQI0syZdckHmB6UzYhVtnOzg04yXEMhambB3QcjT1lGwlia0T7N8AwcOZAUCQLtGH+MfjwdIySfjnoBSYY/6Fo56u4FSGOPaPpKPoP3795f7t0I+hHr37s2zvzU4CDQTX1sv7iqyf4HaJkt5pQP5fqIlmJUOVgMw2MEJFnnN4G5hjlt5YcjML56FdBmD/zg4qf2qHEtKAFTZ2BA6B1tMTJCbUwjk3MC+9GwcTEuAvaUr2ts5wFXRHSOtlqBkYXOJRGEpRcpNF/d2gKkCA5r8p+jk47bnJIfgTB7QP/AD+MoBlW0L+Fjb47JUzuI+KHKegqdiBXJzkqELXmBYgSgZ+1qPeds3gyVOIiQxDW/Z1c2UQF52Ii6oCtBIYoe61jtaxQZo6pmmoAVvo+QErXnz5uoHb1ImtBFkMpn4ZUoWnIRAZigFU5S03nv37t3CKY1jdna2+GVIezuEQJt5hT6R4yMhLpyv62NM4gmcSM+AwswW3d3bw960ZJ+OeE7higFugZBLnPc9Xs5cbLswBRMTLHC+5yzYFq+9L++asxFLMeLSHkzu+F9McNfczJqWHY1TSdeQkgfITB0Q4NQCjS2kO0/Lq1E7eWQ2lqw+CeHcuXPi0jwhryJHLy8vjaVH9BCl7XvkSXKT5as5c+aIxagPHTt2FNPGHjE1kaOelSeq8AhUa+jqco/AZ599Js7sVqRDZFlJaiVPeg3N8kot8EnP1VU89MFR3MjKhoXCGUHurYsNBhe1Lp6z8MZAl8aVEElq4eeDx1yXjz2hM/F2ZCJ+7/UdOlfGp9RjaqVTgqnrJxSr0dO0ZJnMs/7xxx8a9bq5ualf5tXEWNN+x/zCfJiaySH+hCmsxdWDcpjC1FSJRjY+sJbR758vWsEVLS1NcBwmUBX/0xCwVKJAVbSRukD1aOtvpS4BTOWgtSc5haQSKNStFKIQ9H0hylnmIt3IYAWiLsfBoTueNl+D7RH7kOQzEg5KN2zsFyxK4OHcB6FD+ojpfgFfIjxATD4i4oYf+wXjR+GsmbVGnWHpN5FbUICWjtWzqCJUX9tHWkpEDqyk5ifprS3lkdUcXQkkp3TZk1Suivwor1y5EvQpHezs7LRiGWjPjbmY86BImiFt/sACL+uiRNZpjDk6F+GUsmyJ887+kD9GESjdn+qno/HeoQk4miOtqTWCh86BrtzRpEgKSq9UysrGV61aJS59q+y12i5PpoLJiZw0kJECDkxAIPDNN99USIF4nDnWiRMnqs21CnVq87ju6jf4s9iH2bhOwZgqOIJI34G3TiyB2o+7XV8MDHqcIqDNHuhG27T07MUXXyxXGHo5V+2QugYr4rviFZPduJaZC98GPeBQ+snXVAF/G1vkP4hFPYeemNaoLZB2Ci+FXUcTmQItlXbYnBSD/QkJ6C3biXlkKbqcVSQ+tg0BnMbe+It40aEz9lz8EbcbTMYEdSdScTP2HmBTygGkQ3d0Ua7BvgvLcNn9Y7gmb8aZlOTy5aw2jJqtoDTGmq2daytFwA39G7THzrBtOJX6AgbY1v5b1CsPzyMPDujqrD+bykhRoDf65HBKCGSqcsmS8tcdCmV05Si17CR1okV9oBmJxwVa1iJVQOgHUwjSuoS8mj02RUeHaASH/YMvvF6FE1S4HHcSd+XWeMpUjlOSxvKzwrEm8hhi8nLgpKiHfl490diiaJlcXPJ+/H7vNiD3hHM+7RAtCQWFCdhxcwcuFF/3gs9AuD92648dxgV+jUWeAbDNT8ZvZ9/DzPgLuJwM+NehPkmbNgWFLycnB0uXLi3pVCVitK5XeCtP63mruu+hEk3WelFyXihVHjw9PUGbwDkwgcoQeJw5Vlqy9/rrr6st9FWmztov2w5NrK9g662tmOg6FJbIw6HwXUi19sLT6VE4IBEgMfUifrl3BlmFhfAwb4xBXl3gUWxmOzRuG7YmxMLRpjFK29dKz7qJzXcO407xdcO8ny77ACxpp7pR6fLI6tZF1rKks5LS+mi8g4ODy2yGpxdz9L1I410jQdkBp69OxpK0NHg798TMZuR3och6ZUn9ZujX/B1MK1iN36/NxqorBYDSCc82IQe0SrzoPxbXsn/C12fewGqHVnjb1QlLNF5qFdUU6PYMpjW+i6V3fkCjW6bwduiB+fSzaPEMxtv/ixXX38aeB0NxpJ10ZtYNE1u+hQeX/sZzu4ZAZqJAT68xmNKiPDlLJNaFGCsQdTwKHdyehW/0/3A9NQoDbJvUcusJuJgYj+4N30Ufa/3yB0DWYmgtprBpjh7YyGxb7T9E1+yQSE3ISuOTJk0SfVXcvXv3kY0uXLhQPCeN09pywU5/zZmTtUQPj/Y4e3UDTqaMxECbQuy+HwZn80A0k8XgFJn1oqnWzHMYum8GouV2cDRTIC0nEevjQ7G562RYpR1G18MLYC13gK2sEPezU9QzF3RdXv4tvLfjIxwrvi4rNxHbEu/jn46vFVVc7v/W6OjaEA8y4xGRegmr49Ngp+iBtrWgPJB35vv376uloGn8Vq2qtkmNNnUKJojpPiYThIYaaPaFnBUKgfoeHR0tJPmowwTo+5X209R2kL4QKd0WPWCePHlSbVGn9DmahSa/QFW1QFa6vtpJ26KbY2Osj9uMe9mD0dg0Dr/ExaOlW284pkcVN6lCZsouPHtoCdKUTrA1VSEpays2Jd/HxvYjoHqwAc+GrIW90hHmqn2Io+/Z4rfbuXnnMWbvDNwqvi49eysOZ+Xh10DyJFI7Qfo7Vd0WaI9XaQWC8sqzWEj7A+l7t8aD0h/Le04XkBZX74bl/UtWf6gzzTwxttXnGFvO176ldQDm9Viqse9hUnFNpVeKjPWfjrFlpsedMaXHKkhdaO4aWNJ+C4/B2OQxuJyul5Kz1KqVci6o0yxWIOoUN+Dq0Bq7e6+oo1adMbN72WUyddR4tZsJCAgAPRwLG03pAfrBgwdYu3ZttevWdgVkg58+FIS9FBSv6FIYqQUpwZwsrbevbmjh2B5Kk8M4lhCKPpamOPwwCs0bvwHnBz+LVe+J2IlwEyVmtvsOo11ccC7sW7x44zI2pWfDNZpsrysxrd0PeN5JhcWnp2FhsVPd03d3Yr/kumvh32LwjTM4XzBSXIcqNiKJhN5bixHFZvBoXWg3j8YQVgtIilU6SntspMvQYmNjERISUul6aEZB6sytbdu2oI3NxhAEU7PUVzIPWxP3oHa5FSL03kb8FB+Pj/1fh1vhVXx5bRda1BuFMR4lJpwfL2MKtlxbhJ15XfBdy15gswnl06K/PUMwx9rTtQVWRd3Gmcz7kBecw6XcLLzm2h4xkcUPiIXZWHJlK5LMfbG629dob26Cf8+/h/HR+7A7ayii7xyHhXlj/NZtLrxldzHj8Cf4R40sDzuu/IXrkutOXX4PI6MO4XotKhDlj1bN5NKYSy0WCrUuW7YMnTt3FpJ81BMCrEDoyUAZo5jOzs4gm/K0kVrYwEwbVGlzpmBm1RC4SB9ipcufaOOsYLKOlKhHzVSU94VMa0fJBK3gNKyinJxtm6CHQo7d0YcwBvG4UlgPSxs0QXjx/giq53ZmHMxMzdDc2gPknautc2PgxglcTimAT/E5H2sXmMly4WtuDRQrEHcz4qFS5WDGyTdR8s7aDqGZBWj7GAHbeU9AuPcEICcGP1+cie/v/oLplk0wv0kASrZ6P6aCR5wiPuWxK6+4YCpSOEd2yMlal7EGMjtLm7ql1lJottAQFKeE9DDsvX8HY5rmA5lh2BNzChmWPR+tQBSk4WRSBJwsG6KJJRlKuI+dd05hr8ocCaxAaPyJGKI51obO7fGU6Z/4X/hpyBT7kWrqj57OHhBNKRTm4mxOOuzN68FXYQH60url6Q9EH8DV5FzczU6BvXkjuCuUsDTzQFNhSWdhLq5mpiArOwov7RuhwfFaVrEbAo1c3U+U/r5t0aKFaMq65qVXoKljANLlLuAH3ZqnSzUy19rhyrXWIIHjx4+jYcOGoIcWChXdjFeDImilqp9/LnnrTwqBsKGMlKn//KfYPNwjJKM9JMSr0hvPzTzwToMG2HXrKF5Nz4a/55sIMkfRJurithzl1ihUxSM9n5z7WOBBNnk5t0R9C1NontNczWsvt1Zbufi/NrPQ0VqwQiZHPVslbiU9oiPSbKUnBjYbhBVxPyMx/R4KUD0FQlp1eXFa1z906FD1KdrcXt60e3nXGUNeu3btNEwak2fsqu4L0WVevs6DsKFnJ1gpPR4tZvZNjDo+A4OaT8OPTZ9SO6Ob3WMhPlDZw+vRVxnlGXIwKMwoSwHotTlWRRO8Xs8db0X9gW9M09C18X8RIHUMaCpDPVMFbhTkIF9tpUeG6IwY9TolTwsZcmRKnM4VzkmomMjgKjOHlWVTLG4zAY6i93ElGloA5Xsnklyv41Eyy1q736lOmNThax2noN/isQKh3+NnFNKTjXuyoS+1D02zEqRY6Ir9+9oeCHIMJjgHIw7kk0IIgsfuo0ePIjxcbS9J7YlVoZD+igmln3Q0RQufYWgc/g1u5csxyt2/zPKi7q6tUXD3Khbe2ICHrg7YePMo7K0D8aKDAg/TfdXnVt3YgGiHLCyOuQkoWqob9XNsBpubx7Hl7h7Y1A8EcmPx+/1orOkorCYtR7bsvRh57jSC6neEMzJwNHwjEuCIN7zaqk3flXNFpbK6du0KMqNLgZYekUd0Do8mQGZqe/XqpaE8DBs2TAvKQxoWHRiJNfJe+I9lIn6LuQAzeQC+aPcJnnNWqs9tVnZBy5xr2JrWEJuGTgNi9mDRlbX4lzzSyj0wxu99vN4gEJaqVKwNm48fbp5Buqk7guyEV8DAvaQj+M+JZejtNwffNW6NqzHb8M2Nv3A0LQn2ytZY3O1zbDlTNJ+2/fqX2B7uhrXdpmLjyQ/xj2wYQnqNgVNeBL67sARLY6+qwfrbd8YnrSeip20hVhwciWUmXTDMJha/xtyBpTwA33aaht52CoTFbcb4C3/iXl4+OnmOwqJ2L8Lx0UOjF2dKKw+6YI61+uBkaNNwCByjViAR9fB6Aw+YQWL439QCozwa45/rl/B52A48Y5mKhVdvwMNxMAbbK3HezhO/3b6GH8J2oLHqfImFHxM5Ork3wY/x+7D67gk84+wJZN/G4kRT7O80pvpi10ENtBeMVgxERRXtB/Hz81MbkagJs6x1ID438QQC1VkB8ISq+TQTqFkC0qU+9BaelpVwgNoeOn0hS/0EUJzWpFcpmHdDF3dvyGUKdLYva73L2+MlhHafhJSEnfj04nLct+yM3zpPQ0OZGdp6j0dwky4Ijd2Anx48wKJmQaII3m7P40SPyfDMDMGnFxbi04gjGOz/JqweZxbWvC8+8XRFMJW/8DMOyX2wps8ivOlc/V0QNKtAShexow8rD+JQPTJChgykntlp/w3ZcNdWSEr8F3vMR+Bkr2/glxOKT44vQ1ixMBEJx3HFbTJCh86BR/J5jDu/HKlOIxA+ZBO+r++C+dcW4lJGKiKSDuOHW2fQvv7HuDJgMrJzhM2vZXv19dVlCMtviiP9/8Ya/1Y4m5WJue2LzNfSDET4Mz+jk02JvxcgDSuOvY+lsQ/wbe9ghPf5BnbJJ/DGwQWinEkpx3HAfDzO9ZoE25xQvHn9KJAbgckhv8LN6SWED1yL7qoU3Csrjt7mkHU5Wq5pKA+StjZB8LK2h4dNfbSUF5vAFkdHhtZNp2JLm2E4dfsXfHp5PRzqvYY/Or4FJ8jRJ2A65tfzxpbw5dhX4I19zQVP7jIENHwbWzu8jPTYTUXfmTFX8EFgycsjsQkdjdAyZKk556tXrxrMmOso8joVi2cg6hQ3N1YdAmRrnh5gPv/8c7EaskRE3qotLIQlMeIpjlSCwJjOwZC+05rRfpFkn4IVJnT9qdiWdVGlFg7d8e8z3cttwd/vM5z1+0w8Fy7RQZT23fG/PmWvE/c5iFeVRNp5j8M273ElGRzTCoFdu3ZptPvuu+/ip59+0sir84R5V6zyDwT99Q+o54YT0Vdwt3jPDSy649fic9dSzyO9ADDJj8GcK6uQlZkN5KXgYPpDdEo7hzQV0Mu7C+RmcrxRvxOO37lTtivZu3ArE2jmPwqeSiU8vYZD7aYn49FW1JAVgRWpgJ/XMAyzoir98aJ3I5yIkMgp6cNgx++xOCMWD2Ud0cMOWJy4F9MuZ+HZJsNQNI9XVixdzyE7/4K1J901x1p5inODgjFXuMzMDv88/ZuQAuCBb4cG41tJToDXcFz2Gi7JKYkObfc1hrYT0q8ivKkQB7w9hmP9oLLXlbb+U3KF7sWke/t0TzqWqKoEWIGoKjm+TisEyNrNw4cP8cMPP6jbv3btmvpNO39BaWU4uFEjIXDv3j0888wzYm/lcjlGjx4tpnUhUuQFNhM5ghNYpT3ciwXLL8iBCkBY0lnEpBRNvNez8EB9pTnyk3NBlyhlJUuXyu1PXjrIwqaVrBImsQvzQYYpvSR1K9XXS+QsrzGZAz4JWg7l2YX4NXYL/oi8gnUD5+IpcR18eRfpZl5QUJB6tkE3pWOpmAATqCoBXsJUVXJ8ndYIkPnT0ral169frzV5uGEmYMgEyClc06aSV6KA2tEj2ejXesi9hh+j7iI3JxS7HiQA5kEIVL/p15TM0aohlCZAV8+XcbjvShwO+grv+3RGRws7uNh4g1x6noy5gIKCWOxLuKF5sZAybwpnGRAasxeJBcD9hOP4/UHJWvfkDImpMuEaCy+MsgCux57BCfIhmROK9dHhj5RTuAz5MZgbGoKRreZhZQA5lLqK88maThnFshxhAkyACWiBAM9AaAE6N1l9AtOmTVOvvRbMSJKH3/r162vsA6h+K1wDEzBuAhcuXFA78xIoyGQybNu2TXTwKORr75iPszemosX5NJiZOeDtwNfQUNP4l1q0xo7dMcrlBDZGLkGjO4sAUzk87NthqacKzZx6obPtSWy9OQMb71iimfIRyyHlLfG8VxusiPwbHbb/DYXcAW+0XgBYNcRkG3vMj1qGJnFbsK771BIcps74ou0rOHQ2GKP3DFHnO1k0xZz2b5Qrp3ihqhAP4/5G59s/oxAyeNkMRF/nSsx8iBVxhAkwASZQOwRYgagdrlxrHRD4888/1T4ihKZoqnzTpk2i6U0hn49MgAlUngAZKpBuzKcayPRiYGBg5SurrSsUgfhf9zFQZWdCJrOBj5UDgEL8p/1C9DaRbGY2c8bkDjMxKisByQX5gKkZnCw84GImhyl8sbjzN2p7/PkyczSQK3E3JwMNLa0gt+iOP3r4w9q8yIzrhIApGOQTh7RCQG5mAx9Las8EE7r9gKcz06EykaOhpRvqdVqIsbAv8n7rNAIbuj+N2Nyi2Qo7pRs8lDRNUoAX2i9EV5UVXIr5vNJ6IZ5R2cNO7oi53X7AmznpUMEUrpaeem+BqbZuAa6XCTAB7RBgBUI73LnVGiDQrFkzHDlyRG1WMi8vD4WFhSCrMDt37jRqJ181gJarYAIgBZ28dQuBlHOdUh6KBbNWuqOexst5GVxsfMWHckF+yCzhad0AnmJGSUShdEITZYlVN38xbodmdnYlBU0t4WUjWMkpycrmggMAACAASURBVIbcFc3sSiyDWdj4arTjYOEC+mgGGZysfVHSKuBq7St6WTczd0Uz85I6Na/lFBNgAkxAuwR4D4R2+XPr1SRAb0jJNr000GZPWnphzGHNmjVITk42ZgRl+k6WYJ5//vky+ZxRlgD5Fpkxo8RfOJWQmlEuewXnMAEmwASYgDERYAXCmEbbgPt64sQJjd599913GmljSJS2qe7g4KBev/7hhx8iJCTEGBCU6SP1nT4mJiY4dOiQxnlWsDRwiIm3334bpIAKISAgACkpKUJSR442eP/pYIT3m4J6OiIRi8EEmAATMCYCvITJmEbbgPtKFmGkHi/XrVuHnJwc/PXXXzA1NR49+cqVK2qP3bSkiwJZ0KHPwoULyx39O+XZuy+3pO5n+vj4VEhIuk8+/vjjCpU1tkKkbC1dulTstpubGy5fviymOcIEmAATYAJMgAiwAsH3gcEQCA8PR//+/XHgwAF1nzZu3IghQ4Zg+/btBtPHJ3WEHOvt3r1bvZl80aJFTyqOij50P7EiPSnQsWNHtfdp8mPAQZPAm2++iRUrVoiZ3t7eOH36tJjmCBNgAkyACTABgYDxvJoVesxHgyVAD4X//vsvWrVqJfZxx44doPXcxhSefvpp/Pjjj2rnTWSZij70JtkYA5kdpf6/8cYbah40G8PKQ9k7ISEhAVu3bhVPkIECmp1ydnYW8zjCBJgAE2ACTEAgwDMQAgk+GgyBo0ePol27dggLC1P3idZzl94fYDCdfUJHDh48qC4RGRmJxMTEMqXbtGlTJk8fM2bNmlWu+V5SIHTRcpCuMe7QoQPi4+NFsaZMmSLGOcIEmAATYAJMoDQBViBKE+G03hOwtrYG2bCnjbNCsLS0xKVLl9C4cWMhy6iODRo0AH1KB5VKVTqL00ZEgJTKLl26ICIiQuw1baLW9qzd/aTteOPMb7ialYv323+LDz3r4O827yE2xp4XOVDE0aIpurvkY+LeidilGobT/cbUgD+GLPxxcgKmJbth34BvUbGdOxpiPSFxD8/tfBuZLh9gS/u+eIRbvCfUwaeZABNgAo8nwArE4/nwWT0msGrVKowdO1bdg6ysLDRp0kS9jEWPu8SiM4EaJeDkJPVCALXp1pkzZ9ZoG5WvLBErz/yBq6YtED70i8pfXsUrUtIv49MLmsYGWruPQyeXjrBTAFBZo8I7Z7JvocfBqfBwfRl/tn2hihLxZUyACTAB3SXACoTujg1LVk0C9BaVFId33nlHrKl+/fpqq0T16rHxRxEKR4yOAP1d0DI/afjiiy+gfeUBOBO1FSfysmAui8FXV1bjRd/nEBG1AQn2PeGRfhZHM50wIaAvHLLCsSbyGGLycuCkqIde9YPQ3NIKSWmXsTjyLAZ6dsPJBweQkGuL/l6D0cT0En6NDkVGgRtG+w5AAwvSCkoHUwxq/jl+bPqU5EQqWrkNhYVZYyiQiT1ha0vJ0gO3orZjX8pD2Fg3xNj6vXAuahNS8vOQl3wKc0IfYqjPy2hlZS2pUzOakxuF4PD9uFaQB1elN17w7gsXeTJ+v7EBybLWeL1xB1giD+ci/sL2XGe807Q/nJCJ8xHbsCc9BaamVhjQ8HkEWvF8gyZZTjEBJlBbBFiBqC2yXK9OEKDlGGTO9aOPPlLLEx0dDVIieOmOTgwPC6ElAjdv3sS1a9fE1sePH4/Zs2eLaW1GopKv4G5BHnILE7Av7jR6NuiLkKitWHV9K5RyBzibPYURjTwweu9URMvt4GimQFpOIn6LCcHGbtNRkHkbv4RvxC93dsBdaaU+9/fdjcg2MYeLHEjISkJIvgKbWw2oYDczsP/uVuxXOuDDxg3KyBJgG4VJFzbBRekCmSoEFtZ+ME2+jKzCQuRmRWJvbALa13vhkQpEbv5ljN/1OS4oHGAvkyEjJxg7UxKxuf1LCIk/hFN5D/Bcw/YwV8Vh4Y2tuGnVFV807Y/VZ2bgq9g7cDW3g6ogHctuHcLyPv9DL8sKdouLMQEmwASqQYCtMFUDHl+qHwTItv3vv/+uISxZKeLABIyRwPr16zUslRGD5cuX6wyK5wOn4j/mtnCw64IDvf+HrjbCW3UffNljMQ72fRdRUbsQbqLElHbf4XCf5VjZpCPup4ZhU6rg8M4MzzWfhH19FmOSW0Okwwrvt/sS//b+L160AC4nXMXDcntciDP3/sKEkC/VnyWxMeWWAkpkkaXfU5f5sNVX2NdzLgbbOWN84HS4K5Ro6fEiDvddiWccNJeKSSvdf/0fhMhd8G2nhTjcexlWNmqGy/EncQlAH4dGSMy5hWtZyYhPv4kreVlo49QeQBj+ir+OLo0+w/7eK7G943twksVhS+Q5adUcZwJMgAnUGgGegag1tFyxLhF49tln1aZMBUszEydOhK+vLwYPHqxLYrIsTKBWCZw8eRIvv/yy2IaVlZXaL4aYocsR5254oXgZUFRGNMxMzdDc2gMwAdo6NwZunMDV1FwMVesbJmhi7QoLEwsE2FoBD/LRzs4JCtNCuNKvXl4ush/R17zCHKTmZajPZhUWlO8uSSJLJ4+eaBV/EzNC3sQGh96Y22HCI2ouPzssMwl5eQ/w7uFXJQUccTETGOLRCoo7lxCcEA0neRgyCxXo5tYSSNuF+ALgxq3ZaHGr5LKErDgAniUZHGMCTIAJ1BIBViBqCSxXq1sEyDJTSEgIGjZsKApGTuaWLFmCCRMq94MvVsARJqBHBMjBIjlalIazZ8+CfD7oW7BT2KJQFY30/CwAFniQTfMJ5vA0r+5Pmik6N3il1B6I2MficXXoij+6Ncf5+D0YdW4DPg/vgI2NKu53xdnMAgqFG75pMwmNzYV9GXJ4WQI25p0xxGIVtkSehLNyP3IseiHI3hpQuYF2VLRsOh2TPUpmN2wULgCEWZjHis0nmQATYALVImAUS5hu3bqF3NzcaoHii2uOwIULF2quskrURGZMad23g4ODeBXtkVi3bp2Y5ggTMEQCu3btQq9evZCXl6fuHjnTO3LkiF4qD9SBji7tUFCYg4U3NmDj3WBMvHYUtpbN8YLjozcq19a4bg1bgm/vXEJ0TjZcYQpfK2dApkQ9E1NEJp7Exsh9iMh+1HwH0M6lKZS5Cfjz7iFcS7mNa/cP4rOrf8OGBDb1wHMNW6IwdQvWPMhDkG9feNKvtqwF/KxtcCHid5x4GIZrSaFYFfodVieU9fVSW/3mepkAEzBuAgarQGzevFkc2WPHjiEzM1NMc0S7BLZs2SIKIDg6EzNqOdK8efMyDtWkVppquXmunglohcC8efM02qUXKt26ddPI06dEfdehCO0+CSkJO/HpxeW4b9kZa7rOgZ9CWefdCHLpgjPXF+LTK1vg7/YCPq/fDJB74Y82fWGaeU1tGjY8h5ZClR+ae43F/s4ToEzcpy77adQZjPAvMj9NVzTxHAxPGa2k8sT4esLyJEcs7TwHo+xysDD0f/j06lpker2Hd7xKZljLb41zmQATYAI1REBlwIEsdwufCRMmGHBP9adrU6ZMEceExkZbYdWqVRpy9O3bV1uicLtMoFYJjBw5UuNenzFjRq22V17lwvcwHQ8cOFBeEc6rAwLGNg7S/tYBXr1tQp85zZ8/X/x+a9iwod6OQVUElz7HaKPvBjsDQfoVrfkVwtKlSzF9+nQhyUctEPj+++8hfRO6adMmLUhR1CT5iAgMDBTb37t3Lzp16sRL3UQiHNF3AoWFhRg+fLjGEj3yf6ILvh70nS3LzwSYABMwdgIGrUAEBATA29tbHOM5c+bgzTffFNMcqTsC5Idh8uTJYoOWlpZ46impsybxVJ1FLl26hJdeekls79SpU2U2mYonOcIE9IwAGQnYuHGjKHXHjh0RFhYmpjnCBJgAE2ACTKCqBAxagXB2dsadO3c02KxYsQJffvkltLWRV0MYI0iEhoZiwYIF6o/QXbKElJGRAQ8PDyFLa0eyid+1a1exfdqTQbMTHJiAPhOgWYYdO3aIXWjVqpXaAzsp7hyYABNgAkyACVSXgEErEAKcWbNmCVH18YsvvlBvIGzdurVGPidqlgDxpYdzwQu0ULuuPaAfOnQI9LZWCFu3bkVaWpqQ5CMT0DsCtGRTCE2bNtUfXw+C0HxkAkyACTABnSZgFAoE7X1QqVTw8/MTB4PegF+8eBEmJib8qSUGxDc1NVVk3qhRI/XMg66twZbJZCClQQhJSUmwtbXlWSoBCB/1hkBkZCRo5lVwmEiC37hxA+QHhQMTYAJMgAkwgZoiYBQKhADr6tWrQpSPdUzg/PnzIH8curyEQrrpnvC0adOmjilxc0ygegRoeeDDh+RUrSisWrVKiPKRCTABJsAEmECNETAqBYKo0UwEvW2eP38+PD0Fm9o1xpMrkhAgxvTZsGED9GG5WM+ePfH7779LegAolUq1B2uNTE4wAR0jEB0djfr162tItXjxYt7Po0GkphIFuBK9GXOub0Fyfv5jK41NPo05oStwLJUdvD0WFJ9kAkxA7wiY6Z3ENSCwsN79ww8/rIHauApDIjBq1Ci10jBixAi1skkOtw4fPgyyYMOBCegqgdLKAynu5GWdQ20QKMCdB4ex+r4JxvoMgL3Zo39GE9KvYvXtrXBx7ouuto61IQzXyQSYABPQCgGjm4HQCmVuVK8IkO384OBgUWYyPztt2jQxzREmoEsEfvzxRw1xli1bBn45ooGEE0yACTABJlDDBB796qSGG+LqmIA+ERg0aBBoPfndu3fVYn/11VdqnyLjxo3Tp26wrAZOYNu2bZg4caLYy2effZZ93Yg0gLTM2xhxeCLq1xuHhIe/41KqDH29J+BF+7uYef1vxGQ7YXyLj/GJb0vIVZk4F7kJn97Ygjs5WXBXNsYY/7EYU78l5IVp2HzrW8y4cQEqZVMMsCTjEHbFLeUgPOEY5p5dioPF101u8ykGu2jfTLUEBUeZABNgAjVKgGcgahQnV2ZIBE6ePKnRnfHjx7M3cw0inNAmATLVKizHFOSgpUscyhL4N3ovXgqYg0le3tgb8QPevv0Qczp8if42wKrbv+NBXi5u3d+B0Zf+RD3P0Qju/g36KzLw/aX/YV92Om7d34lPblxAL58P8Evr7jiTHi82EvfwMF48Ph/5xde9apuBzy/9pK5TLMQRJsAEmICBEWAFwsAGlLtTcwTc3d3VZmfJ/KwQyJt5cnKykOQjE9AKgb///ltjj4ODgwOuXbsGHx8frcij6412qT8Sw5ybo5dbKwByfNz8BfR08MNAB0fk56XjbmEezidcQL6pBcZ594e/gz8+aN4B+QUJOJmUjfMPL6m7+IJ3D7R3G4iPPZqIXb6SeBEpABorzHAtLQZmFo7IyYzErqxMsQxHmAATYAKGRoCXMBnaiHJ/apQAmZ0l87PkL0QI9LBG1rw4MAFtENi8eTNefPFFjaYTE9nKjwaQRyQsZY/2xJ2elwFTExNYyJTqq+0VtEQpFwm5BaBzFIrO5WrUnpFXpCisvrFEku8MK3nJd4bkBEeZABNgAgZBgGcgDGIYuRO1TaC0j4hOnTrVdpNcPxMol8DKlSvFfHNzc1y5ckVMc6TqBJraNUJ+YR6OJkYBhVn4K+o8YOqCXo4W8LIqMvl9MTEKObn3cDT1gdiQp1XRXof3W36F8KHBCB+wAvPbvIJ+CiuxDEeYABNgAoZGgGcgDG1EuT+1QoB8RDzzzDPYuXOnuv5Tp06pvVXHxMSwl99aIc6VliZAJoV79OgBuveEkJWVJUT5WE0CXd17wuPmCSw+9y4WnVPBzNQcHd2Hob+1LVRu3eEWHoJ5597BPFMFnGQmgKyowfaefdDrxiH8fHkaFl1SASYy2Nq0xQ63rtWUiC9nAkzgSQRsbW3FImT0hGZon3vuOTHPkCNjx44Vu1falLd4ohYjJipei1GLeLlqQyPwwQcfYNGiRWK3yFv1uXPnxDRHmEBtEXj66adx8OBBsfqXXnoJ69evF9O6HJEuAaTZPFLI6yLkF2TgdMJlKCyaoI2tE7JzYnE0MRJNHALhY26OuJQruJRViPYuLeAoM0NB7kOcSLqDzMJ8WClcEWDvC7tiRSEj/RZOpyXAROmAVkpThGTkoLuzHyxMZUBeIi4m3UZ8QT4gM0cTez80VCiRnhWJk8kxaOLQEj7mj14+VRcsqA1tjUNd9a90O9L+8qNOaTolaX3nJJV/06ZNRqNASPutjfubFYiSvyGOMYEKERg8eDC2b98ulg0KCtJ4sBNPcIQJ1ACB/Px80D23e/dusbannnoKpa2EiSd1MCL9oatLBUIHUWhVJGMbB2l/tfGApdXBrkTj+s6JXkgcOnRI7PH58+fRunVrMW1oETLk0r59e4SHh6u7RoZeaK9mXQfeA1HXxLk9vSdAtvffeustsR9nzpzBjRs3xDRHmEBNEqBZLqnyQD+W5B2dAxNgAkyACaDMC7xu3bqBlhcbapAqD9TH9957TytdZQVCK9i5UX0nMGDAALELGRkZaN68OY4ePSrmcYQJ1ASBCxcuIDQ0VKyqf//+oDf4CoVCzOMIE2ACTMDYCaxdu1ZEQL/J9erVQ0hIiJhnCJH09HS0bdtWnHmgPr322mv48MMPtdI9XsKkFezcqCEQoGlEMukqDUlJSbC3t5dmcZwJVJmAdGlBq1atQAqFPgZpP3gJk/ZG0NjGQdpfXsL06PvOUDhJ+yHtrSGMPSkJCxculHZLrTysXr1aI68uE2yFqS5pc1sGRYAUBVpaQm+FhUAKBVlqks5QCOf4yAQqSuDy5cvo2lXTis/kyZMrejmXYwJMgAkYHYGUlBT88ssv+OijjzT6TorFxIkTNfL0KVFacSDZJ02ahL59+2q1G6xAaBU/N67vBPr164cTJ06ozWvm5eWpu7Nr1y5WIPR9YLUof1xcHFq2bClKQD9+f/31F4YPHy7mcYQJMIHKEYiIiIC3t3flLjKC0sTFUAKZdKU39eSnifZBFBQUiF0r7yFcPKlHERcXF/Tq1Qvfffed1qXmPRBaHwIWQN8J0JeVv7+/2A36oho1apSY5ggTqCiBO3fuICAgQKN4cHAwKw8aRDjBBCpGYMqUKWJBHx8fMc6REgJSLlJeJSX0L0a/yTt27MDrr7+uf8I/RmJSHi5evKgz5rt5BuIxg8WnmEBFCdDadNpILVhjWrduHby8vDBv3ryKVsHlmAD279+Phw8fiiRWrVqFQYMGiWmOMAEmUHECZO5YGu7duwdtONySyqBLceIhDaV5Sc/pW5xWB9Bn5cqV+Omnn/D333/rWxdEeaX+f8RMHYjwJmodGAQWwTAIJCYmomHDhiBLCUIYMWIE/vzzTyHJRybwSALTp0/HnDlzxPPka2TgwIFiWp8j0s2NvIlaeyNpbONAa+L9/PwQGxurhu7p6Yno6GjtDYCOtUyWigRzpx4eHrh27Rrs7Ox0TEoWR1cJ8BImXR0ZlkvvCDg6OiItLQ1kLUcItHZdXy3nCH3gY+0TWLx4sYbyQLNXHTt2rP2GuQUmYMAE6GF45MiRYg/pYdnZ2Rm0z8iYQ2RkpJqDoDwQC+LEyoMx3xWV7zsrEJVnxlcwgccSKK0wkCMwDkzgUQRmzpyJd999V+O08AOvkckJJsAEKk3g+++/x/z588XraIkgvW035kAz5dKlksSHOHFgApUhwApEZWhxWSZQQQJz587VKEmbrLOysjTyOMEEpk6dilmzZokgaIlFVFSUmOYIE2AC1ScQFBRUphJazkWbbDdt2lTmnCFmUD+pv9JlbEI/teWITGifj/pJgPdA6Oe4sdR6QIDsNP/www8akrKjOQ0cRp3IzMyEu7u7etkbgXByckJCQoJBMpE+tPAeCO0NsbGPw+zZszFjxgztDYCOtUwvL2jvFQcmUBUCPANRFWp8DROoAAGaEv7vf/+rUZLMy3FgAkTg6aefFpUHS0tLnDt3jsEwASZQiwToYXnZsmW12IJ+VE37q2hGgpUH/RgvXZWSFQhdHRmWyyAIkEfMN954Q+wLmXk9evSomOaI8REgK12tW7dGSEiI2HlSJho0aCCmOcIEmEDtEHjzzTehUqlAy5rKW9pUO61qv9ZGjRqJfT516hSee+457QvFEug1AV7CpNfDx8LrCwHp0gFTU1P125+hQ4fqi/gsZw0SaNasGcLCwsQa33rrLSxdulRMG2JEev/Tg8ytW7cMsZs63ae1a9di9OjRooy8lKwIRWmjFyIgA4uQyVZyRMaBCdQUAXYkV1MkuR4m8BgCR44cUbufz8vLQ2FhIZ599lls2bIFrEQ8BpqBncrNzVXfA1Ll4eWXXzZ45YGGkfb+ODg4qEc0PDzcwEZWP7rz4MEDUVAyNd2zZ08xbcwRmg3kwASYQOUJ8AxE5ZnxFUygygSkb2KpkvPnz6uXs1S5Qr5QbwgkJyeLD9EkNCmRmzdv1hv5qyOoMfe9Otxq8lrpdw8pEMby5r0mGXJdTIAJlBDgPRAlLDjGBGqdwIkTJzTaYB8RGjgMNrFr1y4N5YE6aizKA/XV3t4er732mji+NPtGJiU51D6By5cvw9bWVqMhVh40cHCCCTCBKhBgBaIK0PgSJlBVAmSFiZZzSMPw4cPVy5qkeRw3HALkFO6ZZ54ROySXy1FakRRPGnBk9erVGg68Vq1axUpELY83eVxu2bKlaO2Lmlu5cmUtt8rVMwEmYAwEWIEwhlHmPuoUASsrK7UJT0GojRs3YsiQIUKSjwZEgKyd0KZhadi9ezeM1Zzvjh07pChASgRZg/n333818jlRfQLENSAgQKMiwZmYRiYnmAATYAJVIMB7IKoAjS9hAjVBoH379jh79qxYFS3xoLe0HAyDAC0TkS5Rk8lk2LZtGwYMGGAYHaxiL8jr7b59+3DlyhWNGkjR+vnnnzXyOFE1AmQWuHQgk6UHDx4snc1pJsAEmECVCLACUSVsfBETqD6BrKws9QZqqVWebt26gSw2cdB/AmTl5tChQ2JHLl26hMDAQDFtzJGUlBT4+fkhNjbWmDHUWd+3b9+OgQMH1ll73BATYAKGT4CXMBn+GHMPdZSAhYUFyLFc165dRQlpRoJt5Is49DYye/ZsDeVhxowZrDxIRtPOzg4xMTEaG6slpzlaQwT69++vdprGykMNAeVqmAATEAnwDISIgiNMQHsEyEoNvZUVAnlK5aCfBMaMGYM1a9ZoCM/jqYGj3ITUzGi5BTizwgT4fqswKi7IBJhAFQnwDEQVwfFlTKAmCXz99dca1ZHH0OjoaI08Tug+gbfffltDeaBNrNnZ2bovuA5ISA+9/KkZBjownCwCE2ACBk6AZyAMfIC5e/pDYMGCBfjoo480BCbvsc7Ozhp5lU1ERERoXOLj46ORNoYErbeXWgCiGR/61GSgzcELFy4Uq3RzcwOZ0eTABJgAE2ACTMDQCLACYWgjyv3RawJr167F6NGjxT5UxzLTqFGjkJGRAXLaxUGTQLNmzdC8eXN88MEH6NWrl+bJKqakS3C8vb1x+vTpait/VRSFL2MCTIAJMAEmUKsEWIGoVbxcOROoPIF58+Zh6tSp4oXBwcEYPHiwmH5cJD09HWfOnNHwM/G48nyuiMD8+fNBMwhVCQkJCSCFgZQ1IfAadIEEH5kAE2ACTMAQCbACYYijyn3SewLSt9nUmSVLlmDChAlP7Be9WZeahS3vgvPnz5eXbfB548aN0/C7UbrDrVq1UpvQtbGxKX3qsWlaEiZdJjZr1ixMnz79sdfwSSbABJgAE2AC+kyAFQh9Hj2W3WAJREZGqn1EJCUliX2k5U0jR44U06Uj5Vn/oSVQlpaWWLx4ceniRp3evHkz6HP06FGEh4drsKjo7EFiYiK6dOmiNsUrVECbqJm1QIOPTIAJMAEmYKgEWIEw1JHlfhkEgdIzEatWrQIpCqVDcnIyHBwcxOzq7J0QKzGSSGnG5LNh5syZT+x9Va97YsVcgAkwASbABJiAjhNgM646PkAsnnETuHLlCszNzUUI69atE+OPi6xevfpxp/mchADNOAQFBUlynhwtrcTNnTu3QkrHk2vmEkyACTABJsAEdJ8AKxC6P0YsoRET8Pf3R1ZWlkhg7969oLzc3FwxjyN1S+DSpUsavh6effZZjU3vdSsNt8YEmAATYAJMoO4JsAJR98y5RSZQaQIvvfSSeM21a9fQv39/Mc2RuiOwfv160GZraZBazJLmc5wJMAEmwASYgKESYAXCUEeW+2VQBOjBtWvXrmKfDh48iAEDBiA/P1/M40jtEqAN1y+//LLYiJWVFcii1VNPPSXmcYQJMAEmwASYgDEQYAXCGEaZ+2gQBA4dOoQhQ4aIfdm9ezfatGkjpjlSewQOHDhQZp/E2bNn1Zayaq9VrpkJMAEmwASYgG4SYAVCN8eFpWICZQjIZDJs3bpVQ4kIDQ3FhQsXypStTkZ66llsjNyHf6IO4E52ukZVJecOIja3crMfZyOWolHwcGxMeKBRZ+nEw/QTeHr7ELx9/TQKSp8sTsemXMY/kfuwMTb0ESVqNpu8VRcWFqorlcvlan8R5HODAxNgAkyACTABYyRgZoyd5j4zAX0mMHv2bJB3aiHQLITUX4SQX9Xjudvf4dPIIq/K7Xwm4q/APkVV5UXgzaNf4VR+HmCixLedW2CYs0tVm6nGdQ/w9anPsT0bgE0PDPMIqEZdT760Z8+eGoV4A7sGDk4wASbABJiAERJgBcIIB527rN8EWrdujd9//x2jR48WOyL1ASFmVivigec9s7Ep5hTuBvZBQwDRKddxqSAPgba+uJwWXVJ7bhw2Rh3A1ax02Jg5ootnH3S0tVOfT804jzV3ziAZjmhkRk/80pCOI+HBOJyVDiszRwzxGYZGSun58uK5OHb1ZxyX+2OcMhw/F00KlFewRvJGjRoFWjomBN68LpDgIxNgAkyACRgzAVYgjHn0ue96S4AebJVKJUaMGIGKek6uXGeV6N24H47EZroN+gAAChdJREFUrMG6+FxMdVPgcsJp5KAZujm5iQpEQf4DfHp8KrZnZMFFYYWcvBQsufUXfu75G7ooEjD+1Pe4nKWCs0KBP7MfAijSEAoLU7Hg2KdYnpqsvi4vLwXLwo9jQ59v4fEYQVPSb2LW3VMYFPA1PO/OAGpJgaDlSsR248aNojReXl7YtWuXmOYIE2ACTIAJMAFjJcB7IIx15Lnfek9g+PDhGkuZarpDSpte8Lcyx77bu5COKATfvQDP+iPgpyhpKSZxPw6mJWC43wwc6rMSf7Z8EfmF2dgSfxkxyYdxKyMVL/nNwcE+S/ClbzvxwoT0s9iUEi1e93frF5FbcAf7Eu6IZcpG0rDp6gJEmPfB+54Nyp6uwRzarC5VHjp27Ijw8PAabIGrYgJMgAkwASagvwRYgdDfsWPJmQAGDRqEGTNm1AoJM1MbdLP3RkzKNoSEb8auXGuMaxQIif6AtLQbSFYBgY5+MDUBfOzowb4QkZlJ4rnmjo1gamIBLyt3Uc709JtIKQT+CP0UTYKHIOjM7wAKEJWVLJYpHclNPIg5cSkY3sAPt5KuIiKvEChIRsjDq0h71G7r0pVUID1r1izs2LFDLEl+H06dOgXaPM2BCTABJsAEmAATAFiB4LuACeg5gQkTJpTpwfTp08vkVTrDxAz93Mnr9X1MvvEvPOxbYIiVuUY1SqUzKCchJ0Wdn5qXBsAETgorlD4nvVBh4aRezPRywBwEBy0s/izA5AYtpMU04im5qQCy8OeVBRh1fDZ+TcsBMi/h1ZOzcSNHo2iNJZo2bQry/8CBCTABJsAEmAATKCHAeyBKWHCMCeglAXd3d7UVJulG6sjIyBroiwm8XDqji9k/OJ4PtHPsBhuZiUa9bo5BaKj8F39dXwXnnACcubcZpqaOGODaBPbyArjK9mLr9VVwyamPP8O2i3sg7M394GNujiMRGxEo6wGzgmTsi96PPn5z0FNTRxHbc3EfhfCho4rTGVhzdCxm53XA9acni2VqOkIewK2trWu6Wq6PCTABJsAEmIBeE+AZCL0ePhaeCZRPYPXq1eWfqGyuojlebdZefVXvel0gK3W9tWUAdnSfA7ecS/j8wkIczXHEyqDVeNbBGU7W3bCu5UhkpezH1PCDeKdpidUoawt//NVtFoLMojHt4o/49GYwnJp8gX5aMQtbqlPFyVWrVoFM5nJgAkyACTABJsAENAmYqGrHhItmK5xiAkygVgkkJydDOgPBf9aVw02+HqTmWhcvXoy33367cpVwaSbABJgAE2ACRkKAZyCMZKC5m0yACVSMAG1KZ+WhYqy4FBNgAkyACRgnAVYgjHPcuddMgAkwASbABJgAE2ACTKBKBFiBqBI2vogJMAEmwASYABNgAkyACRgnAVYgjHPcuddMgAkwASbABJgAE2ACTKBKBFiBqBI2vogJMAFDIZCYmIi4uDhD6Q73gwkwASbABJhArRNgK0y1jpgbYAK1T6C0FSZqMSkpCfb29rXfuJ63YGKi6duCLVjp+YCy+EyACTABJlDrBHgGotYRcwNMoPYJkKKwe/dujYbIrGunTp3w4YcfIiQkROOcMSemTp2qZkKWlkorD3PnzjVmNNx3JsAEmAATYAIVIsAzEBXCxIWYgH4Q8Pb2RkxMDPLy8ioksI2NDS5dulShsvpWaMGCBVi4cGGFxR4/fjyWL19e4fJckAkwASbABJiAsRJgBcJYR577bbAEDhw4gE2bNmHRokUG28ea7BgpXfPnz8dzzz1Xk9VyXUyACTABJsAEDJYAKxAGO7TcMSYAkIdlCtevX0d8fDwjKSYQFBSkjk2ZMgUDBgxgLkyACTABJsAEmEAlCLACUQlYXJQJ6CuByMhIkLWh0iEtLQ09evQonW0Q6bfeegsTJkwoty+tW7cuN58zmQATYAJMgAkwgScTYAXiyYy4BBNgAkyACTABJsAEmAATYALFBNgKE98KTIAJMAEmwASYABNgAkyACVSYACsQFUbFBZkAE2ACTIAJMAEmwASYABNgBYLvASbABJgAE2ACTIAJMAEmwAQqTIAViAqj4oJMgAkwASbABJgAE2ACTIAJsALB9wATYAJMgAkwASbABJgAE2ACFSbACkSFUXFBJsAEmAATYAJMgAkwASbABFiB4HuACTABJsAEmAATYAJMgAkwgQoTYAWiwqi4IBNgAkyACTABJsAEmAATYAKsQPA9wASYABNgAkyACTABJsAEmECFCbACUWFUXJAJMAEmwASYABNgAkyACTABViD4HmACTIAJMAEmwASYABNgAkygwgRYgagwKi7IBJgAE2ACTIAJMAEmwASYACsQfA8wASbABJgAE2ACTIAJMAEmUGECrEBUGBUXZAJMgAkwASbABJgAE2ACTIAVCL4HmAATYAJMgAkwASbABJgAE6gwAVYgKoyKCzIBJsAEmAATYAJMgAkwASbACgTfA0yACTABJsAEmAATYAJMgAlUmAArEBVGxQWZABNgAkyACTABJsAEmAATYAWC7wEmwASYABNgAkyACTABJsAEKkyAFYgKo+KCTIAJMAEmwASYABNgAkyACbACwfcAE2ACTIAJMAEmwASYABNgAhUmwApEhVFxQSbABJgAE2ACTIAJMAEmwARYgeB7gAkwASbABJgAE2ACTIAJMIEKE2AFosKo/r/9OqQBAABgGObf9SxMQPlRczJDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECW0BAbCpDAgQIECBAgAABAgQEhA8QIECAAAECBAgQILAFBMSmMiRAgAABAgQIECBAQED4AAECBAgQIECAAAECWyBtlAVDCV6eagAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "id": "f7f8bab0",
   "metadata": {
    "papermill": {
     "duration": 0.036582,
     "end_time": "2022-01-10T22:27:20.146141",
     "exception": false,
     "start_time": "2022-01-10T22:27:20.109559",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# What is ensembling?\n",
    "\n",
    "In general, ensembling is a technique of combining two or more algorithms of similar or dissimilar types called base learners. This is done to make a more robust system which incorporates the predictions from all the base learners. It can be understood as conference room meeting between multiple traders to make a decision on whether the price of a stock will go up or not.\n",
    "\n",
    "Since all of them have a different understanding of the stock market and thus a different mapping function from the problem statement to the desired outcome. Therefore, they are supposed to make varied predictions on the stock price based on their own understandings of the market.\n",
    "\n",
    "Now we can take all of these predictions into account while making the final decision. This will make our final decision more robust, accurate and less likely to be biased. The final decision would have been opposite if one of these traders would have made this decision alone.\n",
    "\n",
    "You can consider another example of a candidate going through multiple rounds of job interviews. The final decision of candidate’s ability is generally taken based on the feedback of all the interviewers. Although a single interviewer might not be able to test the candidate for each required skill and trait. But the combined feedback of multiple interviewers usually helps in better assessment of the candidate.\n",
    "\n",
    "## Types of ensembling\n",
    "\n",
    "Some of the basic concepts which you should be aware of before we go into further detail are:\n",
    "\n",
    "**Averaging:** It’s defined as taking the average of predictions from models in case of regression problem or while predicting probabilities for the classification problem.\n",
    "\n",
    "Majority vote: It’s defined as taking the prediction with maximum vote / recommendation from multiple models predictions while predicting the outcomes of a classification problem.\n",
    "\n",
    "Weighted average: In this, different weights are applied to predictions from multiple models then taking the average which means giving high or low importance to specific model output.\n",
    "\n",
    "\n",
    "**Boosting:**\n",
    "\n",
    "Boosting is a sequential technique in which, the first algorithm is trained on the entire dataset and the subsequent algorithms are built by fitting the residuals of the first algorithm, thus giving higher weight to those observations that were poorly predicted by the previous model.\n",
    "\n",
    "It relies on creating a series of weak learners each of which might not be good for the entire dataset but is good for some part of the dataset. Thus, each model actually boosts the performance of the ensemble.\n",
    "\n",
    "It’s really important to note that boosting is focused on reducing the bias. This makes the boosting algorithms prone to overfitting. Thus, parameter tuning becomes a crucial part of boosting algorithms to make them avoid overfitting.\n",
    "\n",
    "\n",
    "Some examples of boosting are XGBoost, GBM, ADABOOST, etc.\n",
    "\n",
    "**Stacking:**\n",
    "\n",
    "In stacking multiple layers of machine learning models are placed one over another where each of the models passes their predictions to the model in the layer above it and the top layer model takes decisions based on the outputs of the models in layers below i\n",
    "\n",
    "Practically speaking, there can be a countless number of ways in which you can ensemble different models. But these are some techniques that are mostly used:\n",
    "\n",
    "**Bagging:** \n",
    "\n",
    "Bagging is also referred to as bootstrap aggregation. To understand bagging, we first need to understand bootstrapping. Bootstrapping is a sampling technique in which we choose ‘n’ observations or rows out of the original dataset of ‘n’ rows as well. But the key is that each row is selected with replacement from the original dataset so that each row is equally likely to be selected in each iteration. \n",
    "\n",
    "\n",
    "\n",
    "**“The whole is greater than the sum of its parts.” – Aristotle**\n",
    "\n",
    "![image.png](attachment:8084db48-9d26-412d-a61a-28fff582b024.png)\n",
    "# Load Lib "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "59268e10",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:20.234854Z",
     "iopub.status.busy": "2022-01-10T22:27:20.234003Z",
     "iopub.status.idle": "2022-01-10T22:27:23.955867Z",
     "shell.execute_reply": "2022-01-10T22:27:23.955205Z",
     "shell.execute_reply.started": "2022-01-10T22:22:58.245302Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.775158,
     "end_time": "2022-01-10T22:27:23.956027",
     "exception": false,
     "start_time": "2022-01-10T22:27:20.180869",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score,recall_score\n",
    "from sklearn.metrics import make_scorer,mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error,classification_report,f1_score\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone,ClassifierMixin\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer\n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "#import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "import itertools\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18c06411",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.033033,
     "end_time": "2022-01-10T22:27:24.025482",
     "exception": false,
     "start_time": "2022-01-10T22:27:23.992449",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae408cb8",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:24.099866Z",
     "iopub.status.busy": "2022-01-10T22:27:24.099283Z",
     "iopub.status.idle": "2022-01-10T22:27:24.530088Z",
     "shell.execute_reply": "2022-01-10T22:27:24.530534Z",
     "shell.execute_reply.started": "2022-01-10T22:23:02.277321Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.470265,
     "end_time": "2022-01-10T22:27:24.530736",
     "exception": false,
     "start_time": "2022-01-10T22:27:24.060471",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 204 ms, sys: 60.6 ms, total: 265 ms\n",
      "Wall time: 410 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33dbdf59",
   "metadata": {
    "papermill": {
     "duration": 0.03365,
     "end_time": "2022-01-10T22:27:24.599249",
     "exception": false,
     "start_time": "2022-01-10T22:27:24.565599",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 1-Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3ced820",
   "metadata": {
    "papermill": {
     "duration": 0.033559,
     "end_time": "2022-01-10T22:27:24.666335",
     "exception": false,
     "start_time": "2022-01-10T22:27:24.632776",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5d10cc52",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:24.759155Z",
     "iopub.status.busy": "2022-01-10T22:27:24.757512Z",
     "iopub.status.idle": "2022-01-10T22:27:24.812181Z",
     "shell.execute_reply": "2022-01-10T22:27:24.812607Z",
     "shell.execute_reply.started": "2022-01-10T22:23:02.699921Z"
    },
    "papermill": {
     "duration": 0.112847,
     "end_time": "2022-01-10T22:27:24.812781",
     "exception": false,
     "start_time": "2022-01-10T22:27:24.699934",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ddd1e11",
   "metadata": {
    "papermill": {
     "duration": 0.033861,
     "end_time": "2022-01-10T22:27:24.880513",
     "exception": false,
     "start_time": "2022-01-10T22:27:24.846652",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5e86cbba",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:24.973851Z",
     "iopub.status.busy": "2022-01-10T22:27:24.972094Z",
     "iopub.status.idle": "2022-01-10T22:27:25.016609Z",
     "shell.execute_reply": "2022-01-10T22:27:25.017066Z",
     "shell.execute_reply.started": "2022-01-10T22:23:02.783958Z"
    },
    "papermill": {
     "duration": 0.100775,
     "end_time": "2022-01-10T22:27:25.017231",
     "exception": false,
     "start_time": "2022-01-10T22:27:24.916456",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f06a31ba",
   "metadata": {
    "papermill": {
     "duration": 0.033847,
     "end_time": "2022-01-10T22:27:25.085316",
     "exception": false,
     "start_time": "2022-01-10T22:27:25.051469",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b80c685f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:25.160103Z",
     "iopub.status.busy": "2022-01-10T22:27:25.159146Z",
     "iopub.status.idle": "2022-01-10T22:27:25.161074Z",
     "shell.execute_reply": "2022-01-10T22:27:25.161667Z",
     "shell.execute_reply.started": "2022-01-10T22:23:02.842570Z"
    },
    "papermill": {
     "duration": 0.042421,
     "end_time": "2022-01-10T22:27:25.161848",
     "exception": false,
     "start_time": "2022-01-10T22:27:25.119427",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cardinality : \n",
    "# - RevolvingUtilizationOfUnsecuredLines :125728, high Outlier\n",
    "# - DebtRatio :114194 , high Outlier \n",
    "# deal with outlier + bin \n",
    "PERCENTAGE = [\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\"]\n",
    "# MonthlyIncome:13594 , high outlier +bin \n",
    "REAL= [\"MonthlyIncome\"]\n",
    "# Can be considred as cat \n",
    "NUMERIC_DISCRET_low = [\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "Late_Pay_Cols = ['NumberOfTime30-59DaysPastDueNotWorse',\n",
    "                 'NumberOfTimes90DaysLate',\n",
    "                 'NumberOfTime60-89DaysPastDueNotWorse']\n",
    "TARGET = [\"SeriousDlqin2yrs\"]\n",
    "\n",
    "#also change the type for TARGET to categorical\n",
    "#df[TARGET] = df[TARGET].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "82de2004",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:25.234792Z",
     "iopub.status.busy": "2022-01-10T22:27:25.233916Z",
     "iopub.status.idle": "2022-01-10T22:27:25.235972Z",
     "shell.execute_reply": "2022-01-10T22:27:25.236334Z",
     "shell.execute_reply.started": "2022-01-10T22:23:02.850246Z"
    },
    "papermill": {
     "duration": 0.040267,
     "end_time": "2022-01-10T22:27:25.236484",
     "exception": false,
     "start_time": "2022-01-10T22:27:25.196217",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = train['SeriousDlqin2yrs']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b6716cd",
   "metadata": {
    "papermill": {
     "duration": 0.033701,
     "end_time": "2022-01-10T22:27:25.303883",
     "exception": false,
     "start_time": "2022-01-10T22:27:25.270182",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Imput before Descritaziation : \n",
    "\n",
    "### IterativeImputer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2bc338c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:25.381690Z",
     "iopub.status.busy": "2022-01-10T22:27:25.381010Z",
     "iopub.status.idle": "2022-01-10T22:27:25.385994Z",
     "shell.execute_reply": "2022-01-10T22:27:25.386587Z",
     "shell.execute_reply.started": "2022-01-10T22:23:02.862413Z"
    },
    "papermill": {
     "duration": 0.048624,
     "end_time": "2022-01-10T22:27:25.386782",
     "exception": false,
     "start_time": "2022-01-10T22:27:25.338158",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.13 ms, sys: 15 µs, total: 1.14 ms\n",
      "Wall time: 969 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "param =  {   \"verbosity\": 0,\n",
    "            #\"objective\": \"binary:logistic\",\n",
    "            #\"eval_metric\": \"auc\",\n",
    "            'random_state': 42,\n",
    "            # regression\n",
    "            'objective':'reg:squarederror', \n",
    "             'eval_metric': 'mae',\n",
    "            #early_stopping_rounds=100 ,\n",
    "            'gpu_id':0, \n",
    "            'predictor':\"gpu_predictor\",\n",
    "            # use exact for small dataset.\n",
    "            #\"tree_method\": \"exact\",\n",
    "            # big data :\n",
    "             'tree_method': 'gpu_hist',\n",
    "            # defines booster, gblinear for linear functions.\n",
    "             'booster': 'gbtree', \n",
    "            'lambda': 8.544792472633987e-07,\n",
    "            'alpha': 0.31141671752487043,\n",
    "            'subsample': 0.8779467596981366, \n",
    "            'colsample_bytree': 0.9759532762677546,\n",
    "            'learning_rate': 0.008686087328805853, \n",
    "            'n_estimators': 6988,\n",
    "            'max_depth': 9,\n",
    "            'min_child_weight': 2, \n",
    "            'eta': 3.7603213457541647e-06,\n",
    "            'gamma': 2.1478058456847449e-07,\n",
    "            'grow_policy': 'lossguide'}\n",
    "                \n",
    "\n",
    "#model_xgb = XGBRegressor(\n",
    "       #objective=\"mae\",\n",
    " #   **xgb_params2)\n",
    "\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            #('imputer', SimpleImputer(strategy='median'\n",
    "                             #                         ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "\n",
    "pipe_xgbr1 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "pipe_xgbr2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer2),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1a6e4679",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:27:25.464780Z",
     "iopub.status.busy": "2022-01-10T22:27:25.463955Z",
     "iopub.status.idle": "2022-01-10T22:29:39.339745Z",
     "shell.execute_reply": "2022-01-10T22:29:39.340414Z",
     "shell.execute_reply.started": "2022-01-10T22:23:02.876476Z"
    },
    "papermill": {
     "duration": 133.918393,
     "end_time": "2022-01-10T22:29:39.340638",
     "exception": false,
     "start_time": "2022-01-10T22:27:25.422245",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 10s, sys: 2.51 s, total: 2min 12s\n",
      "Wall time: 2min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "train_final= pd.concat( [train, test], axis=0) \n",
    "#testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X_income=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "pipe_xgbr1.fit(X_income, y_income)\n",
    "train_income_missing=train[train['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "test_income_missing=test[test['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "train_predicted = pipe_xgbr1.predict(train_income_missing)\n",
    "test_predicted = pipe_xgbr1.predict(test_income_missing)\n",
    "train.loc[(train.MonthlyIncome.isnull()), 'MonthlyIncome'] = train_predicted\n",
    "test.loc[(test.MonthlyIncome.isnull()), 'MonthlyIncome'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3d0fade5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:29:39.418715Z",
     "iopub.status.busy": "2022-01-10T22:29:39.417812Z",
     "iopub.status.idle": "2022-01-10T22:32:16.712695Z",
     "shell.execute_reply": "2022-01-10T22:32:16.713134Z"
    },
    "papermill": {
     "duration": 157.336756,
     "end_time": "2022-01-10T22:32:16.713288",
     "exception": false,
     "start_time": "2022-01-10T22:29:39.376532",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 36s, sys: 982 ms, total: 2min 37s\n",
      "Wall time: 2min 37s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "X_NumberOfDependents=traindf_NumberOfDependents.drop([\"NumberOfDependents\"],axis=1)\n",
    "pipe_xgbr2.fit(X_NumberOfDependents, y_NumberOfDependents)\n",
    "train_NumberOfDependents_missing=train[train['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "test_NumberOfDependents_missing=test[test['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "train_predicted = pipe_xgbr2.predict(train_NumberOfDependents_missing)\n",
    "test_predicted = pipe_xgbr2.predict(test_NumberOfDependents_missing)\n",
    "train.loc[(train.NumberOfDependents.isnull()), 'NumberOfDependents'] = train_predicted\n",
    "test.loc[(test.NumberOfDependents.isnull()), 'NumberOfDependents'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aafd2b3f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:16.797076Z",
     "iopub.status.busy": "2022-01-10T22:32:16.796187Z",
     "iopub.status.idle": "2022-01-10T22:32:17.378600Z",
     "shell.execute_reply": "2022-01-10T22:32:17.377493Z"
    },
    "papermill": {
     "duration": 0.630274,
     "end_time": "2022-01-10T22:32:17.378754",
     "exception": false,
     "start_time": "2022-01-10T22:32:16.748480",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add bin data \n",
    "# initializing append_str\n",
    "append_str = 'cat_'\n",
    "# Append suffix / prefix to strings in list\n",
    "num_features1=[\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\",\"MonthlyIncome\"]\n",
    "num_features2=[\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "cat_features1 = [append_str + sub for sub in num_features1]\n",
    "cat_features2 = [append_str + sub for sub in num_features2]\n",
    "\n",
    "# create the discretizer object with strategy quantile and 1000 bins\n",
    "discretizer1 = KBinsDiscretizer(n_bins=40, encode='ordinal',strategy='quantile')\n",
    "discretizer2 = KBinsDiscretizer(n_bins=4, encode='ordinal',strategy='quantile')\n",
    "\n",
    "pipeline1 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer1)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline1.fit(train.loc[:,num_features1])\n",
    "# apply the discretisation\n",
    "train_cat1 = pipeline1.transform(train.loc[:,num_features1])\n",
    "test_cat1 = pipeline1.transform(test.loc[:,num_features1])\n",
    "train_df1=pd.DataFrame(train_cat1,columns=cat_features1).astype('category')\n",
    "test_df1=pd.DataFrame(test_cat1,columns=cat_features1).astype('category')\n",
    "train_final1= pd.concat( [train.loc[:,num_features1], train_df1], axis=1) \n",
    "test_final1= pd.concat( [test.loc[:,num_features1], test_df1], axis=1) \n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer2)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline2.fit(train.loc[:,num_features2])\n",
    "# apply the discretisation\n",
    "train_cat2 = pipeline2.transform(train.loc[:,num_features2])\n",
    "test_cat2 = pipeline2.transform(test.loc[:,num_features2])\n",
    "train_df2=pd.DataFrame(train_cat2,columns=cat_features2).astype('category')\n",
    "test_df2=pd.DataFrame(test_cat2,columns=cat_features2).astype('category')\n",
    "train_final2= pd.concat( [train.loc[:,num_features2], train_df2], axis=1) \n",
    "test_final2= pd.concat( [test.loc[:,num_features2], test_df2], axis=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "74f9c4cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:17.455134Z",
     "iopub.status.busy": "2022-01-10T22:32:17.454301Z",
     "iopub.status.idle": "2022-01-10T22:32:17.459030Z",
     "shell.execute_reply": "2022-01-10T22:32:17.459436Z"
    },
    "papermill": {
     "duration": 0.04545,
     "end_time": "2022-01-10T22:32:17.459601",
     "exception": false,
     "start_time": "2022-01-10T22:32:17.414151",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_final= pd.concat( [train_final1, train_final2], axis=1) \n",
    "test_final= pd.concat( [test_final1, test_final2], axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8dc3e3df",
   "metadata": {
    "papermill": {
     "duration": 0.035413,
     "end_time": "2022-01-10T22:32:17.530032",
     "exception": false,
     "start_time": "2022-01-10T22:32:17.494619",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Final X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5be25b75",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:17.605046Z",
     "iopub.status.busy": "2022-01-10T22:32:17.604293Z",
     "iopub.status.idle": "2022-01-10T22:32:17.606360Z",
     "shell.execute_reply": "2022-01-10T22:32:17.606868Z"
    },
    "papermill": {
     "duration": 0.041646,
     "end_time": "2022-01-10T22:32:17.606999",
     "exception": false,
     "start_time": "2022-01-10T22:32:17.565353",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pour le train test\n",
    "target= \"SeriousDlqin2yrs\"\n",
    "X = train_final# axis=1\n",
    "X_test_final =test_final# axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "03b5c34e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:17.682645Z",
     "iopub.status.busy": "2022-01-10T22:32:17.681720Z",
     "iopub.status.idle": "2022-01-10T22:32:17.702130Z",
     "shell.execute_reply": "2022-01-10T22:32:17.702712Z"
    },
    "papermill": {
     "duration": 0.060693,
     "end_time": "2022-01-10T22:32:17.702870",
     "exception": false,
     "start_time": "2022-01-10T22:32:17.642177",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      101503 non-null  float16 \n",
      " 1   DebtRatio                                 101503 non-null  float32 \n",
      " 2   MonthlyIncome                             101503 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  101503 non-null  category\n",
      " 4   cat_DebtRatio                             101503 non-null  category\n",
      " 5   cat_MonthlyIncome                         101503 non-null  category\n",
      " 6   NumberOfDependents                        101503 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              101503 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   101503 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           101503 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 12  age                                       101503 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    101503 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          101503 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               101503 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       101503 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  category\n",
      " 19  cat_age                                   101503 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 2.7 MB\n"
     ]
    }
   ],
   "source": [
    "X_test_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cf83bbce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:17.780146Z",
     "iopub.status.busy": "2022-01-10T22:32:17.779204Z",
     "iopub.status.idle": "2022-01-10T22:32:17.799746Z",
     "shell.execute_reply": "2022-01-10T22:32:17.800169Z"
    },
    "papermill": {
     "duration": 0.06117,
     "end_time": "2022-01-10T22:32:17.800313",
     "exception": false,
     "start_time": "2022-01-10T22:32:17.739143",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      150000 non-null  float16 \n",
      " 1   DebtRatio                                 150000 non-null  float32 \n",
      " 2   MonthlyIncome                             150000 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  150000 non-null  category\n",
      " 4   cat_DebtRatio                             150000 non-null  category\n",
      " 5   cat_MonthlyIncome                         150000 non-null  category\n",
      " 6   NumberOfDependents                        150000 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              150000 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   150000 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           150000 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 12  age                                       150000 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    150000 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          150000 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               150000 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       150000 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  category\n",
      " 19  cat_age                                   150000 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "af971232",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:17.877949Z",
     "iopub.status.busy": "2022-01-10T22:32:17.877094Z",
     "iopub.status.idle": "2022-01-10T22:32:17.879530Z",
     "shell.execute_reply": "2022-01-10T22:32:17.879103Z"
    },
    "papermill": {
     "duration": 0.043041,
     "end_time": "2022-01-10T22:32:17.879668",
     "exception": false,
     "start_time": "2022-01-10T22:32:17.836627",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del test \n",
    "del train_final\n",
    "del test_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9ef91552",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:17.961536Z",
     "iopub.status.busy": "2022-01-10T22:32:17.960581Z",
     "iopub.status.idle": "2022-01-10T22:32:17.966913Z",
     "shell.execute_reply": "2022-01-10T22:32:17.966444Z"
    },
    "papermill": {
     "duration": 0.051122,
     "end_time": "2022-01-10T22:32:17.967029",
     "exception": false,
     "start_time": "2022-01-10T22:32:17.915907",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e296553",
   "metadata": {
    "papermill": {
     "duration": 0.036701,
     "end_time": "2022-01-10T22:32:18.041205",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.004504",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Num Features\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8ca877b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:18.151688Z",
     "iopub.status.busy": "2022-01-10T22:32:18.150804Z",
     "iopub.status.idle": "2022-01-10T22:32:18.163466Z",
     "shell.execute_reply": "2022-01-10T22:32:18.164369Z"
    },
    "papermill": {
     "duration": 0.086817,
     "end_time": "2022-01-10T22:32:18.164633",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.077816",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c385a253",
   "metadata": {
    "papermill": {
     "duration": 0.07426,
     "end_time": "2022-01-10T22:32:18.308181",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.233921",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# FE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "62ac2152",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:18.440656Z",
     "iopub.status.busy": "2022-01-10T22:32:18.439825Z",
     "iopub.status.idle": "2022-01-10T22:32:18.445242Z",
     "shell.execute_reply": "2022-01-10T22:32:18.445744Z"
    },
    "papermill": {
     "duration": 0.07626,
     "end_time": "2022-01-10T22:32:18.445930",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.369670",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MiniKmeansTransformerEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, num_clusters = 11, encoder=ce.woe.WOEEncoder()):\n",
    "        self.num_clusters = num_clusters\n",
    "        self.encoder= encoder\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans = MiniBatchKMeans(n_clusters=self.num_clusters, random_state=0)\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans.fit(X)\n",
    "            preds=self.kmeans.predict(X)\n",
    "            preds=pd.DataFrame(preds, columns=['kmeans']).astype('category')\n",
    "            self.encoder.fit(preds,y)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        pred_classes = self.kmeans.predict(X)\n",
    "        pred_classes=pd.DataFrame(pred_classes, columns=['kmeans']).astype('category')\n",
    "        pred_encoded = self.encoder.transform(pred_classes)\n",
    "        return np.hstack((X, pred_encoded))\n",
    "        #return pred_encoded"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6bd0a8d",
   "metadata": {
    "papermill": {
     "duration": 0.061332,
     "end_time": "2022-01-10T22:32:18.569757",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.508425",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Complete Pipe \n",
    "\n",
    "1. Features Engineer\n",
    "\n",
    "1. Kmeans\n",
    "\n",
    "1. Sparse\n",
    "\n",
    "1. Poly\n",
    "\n",
    "1. Scaler/Transformer\n",
    "\n",
    "1. imput\n",
    "\n",
    "1. Add bin data \n",
    "\n",
    "## Pipe : \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2f91da60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:18.698028Z",
     "iopub.status.busy": "2022-01-10T22:32:18.696991Z",
     "iopub.status.idle": "2022-01-10T22:32:18.699519Z",
     "shell.execute_reply": "2022-01-10T22:32:18.700001Z"
    },
    "papermill": {
     "duration": 0.068321,
     "end_time": "2022-01-10T22:32:18.700158",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.631837",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cat pipeline\n",
    "categorical_transformer = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "                        #('sparse_features', SparseInteractions(degree=2)),\n",
    "                        #('reducedim',  SelectPercentile( mutual_info_classif, percentile=90))\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            #('general_features',FeaturesEngineer()),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            #('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer, num_columns),\n",
    "               # ('categorical', categorical_transformer, cat_columns)\n",
    "            ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f972531c",
   "metadata": {
    "papermill": {
     "duration": 0.037285,
     "end_time": "2022-01-10T22:32:18.776918",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.739633",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b395e22f",
   "metadata": {
    "papermill": {
     "duration": 0.037443,
     "end_time": "2022-01-10T22:32:18.852100",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.814657",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Selected Models :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3cba3a2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:18.935896Z",
     "iopub.status.busy": "2022-01-10T22:32:18.935252Z",
     "iopub.status.idle": "2022-01-10T22:32:19.290102Z",
     "shell.execute_reply": "2022-01-10T22:32:19.290712Z"
    },
    "papermill": {
     "duration": 0.40124,
     "end_time": "2022-01-10T22:32:19.290890",
     "exception": false,
     "start_time": "2022-01-10T22:32:18.889650",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"3df071b9-e4cc-4687-a6b2-a0508a42612f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"3df071b9-e4cc-4687-a6b2-a0508a42612f\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('preprocessor',\n",
       "                 ColumnTransformer(transformers=[('numerical',\n",
       "                                                  Pipeline(steps=[('imputer',\n",
       "                                                                   SimpleImputer(add_indicator=True,\n",
       "                                                                                 strategy='median')),\n",
       "                                                                  ('scaler',\n",
       "                                                                   PowerTransformer()),\n",
       "                                                                  ('kmeans',\n",
       "                                                                   MiniKmeansTransformerEncoder()),\n",
       "                                                                  ('polynominal_features',\n",
       "                                                                   PolynomialFeatures())]),\n",
       "                                                  Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome'...\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])),\n",
       "                ('classifier',\n",
       "                 HistGradientBoostingClassifier(early_stopping='False',\n",
       "                                                l2_regularization=2.940296779699346e-10,\n",
       "                                                learning_rate=0.015318591316397998,\n",
       "                                                max_bins=137, max_depth=19,\n",
       "                                                max_iter=1537,\n",
       "                                                max_leaf_nodes=23,\n",
       "                                                min_samples_leaf=140))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"9e14d7d9-5e7c-455c-ab32-bcf86588ce50\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"9e14d7d9-5e7c-455c-ab32-bcf86588ce50\">preprocessor: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a4186bb6-a827-45b4-bee4-e097bc5d38c4\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a4186bb6-a827-45b4-bee4-e097bc5d38c4\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"326e9204-a553-40e6-8aa1-c36ff0f3e18e\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"326e9204-a553-40e6-8aa1-c36ff0f3e18e\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"262ba053-ba09-459b-94e7-c70704a6eda9\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"262ba053-ba09-459b-94e7-c70704a6eda9\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"fc27bf64-027b-4341-af09-b6f4265ac0ac\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"fc27bf64-027b-4341-af09-b6f4265ac0ac\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"cdef7eb5-8ae8-4d04-9ab5-2777591d2fef\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"cdef7eb5-8ae8-4d04-9ab5-2777591d2fef\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"9071b3a8-eb61-4461-bbd6-40c3e2bd825f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"9071b3a8-eb61-4461-bbd6-40c3e2bd825f\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"f9da7aa7-edb2-4b65-b772-3b3fb3edb6c5\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"f9da7aa7-edb2-4b65-b772-3b3fb3edb6c5\">HistGradientBoostingClassifier</label><div class=\"sk-toggleable__content\"><pre>HistGradientBoostingClassifier(early_stopping='False',\n",
       "                               l2_regularization=2.940296779699346e-10,\n",
       "                               learning_rate=0.015318591316397998, max_bins=137,\n",
       "                               max_depth=19, max_iter=1537, max_leaf_nodes=23,\n",
       "                               min_samples_leaf=140)</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('preprocessor',\n",
       "                 ColumnTransformer(transformers=[('numerical',\n",
       "                                                  Pipeline(steps=[('imputer',\n",
       "                                                                   SimpleImputer(add_indicator=True,\n",
       "                                                                                 strategy='median')),\n",
       "                                                                  ('scaler',\n",
       "                                                                   PowerTransformer()),\n",
       "                                                                  ('kmeans',\n",
       "                                                                   MiniKmeansTransformerEncoder()),\n",
       "                                                                  ('polynominal_features',\n",
       "                                                                   PolynomialFeatures())]),\n",
       "                                                  Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome'...\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])),\n",
       "                ('classifier',\n",
       "                 HistGradientBoostingClassifier(early_stopping='False',\n",
       "                                                l2_regularization=2.940296779699346e-10,\n",
       "                                                learning_rate=0.015318591316397998,\n",
       "                                                max_bins=137, max_depth=19,\n",
       "                                                max_iter=1537,\n",
       "                                                max_leaf_nodes=23,\n",
       "                                                min_samples_leaf=140))])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params_hgbc = {'l2_regularization': 2.940296779699346e-10, \n",
    "         'early_stopping': 'False',\n",
    "         'learning_rate': 0.015318591316397998, \n",
    "         'max_iter': 1537, \n",
    "         'max_depth': 19, \n",
    "         'max_bins': 137, \n",
    "         'min_samples_leaf': 140,\n",
    "         'max_leaf_nodes': 23}\n",
    "model_hgbc = HistGradientBoostingClassifier(**params_hgbc)\n",
    "\n",
    " #Private Score 0.86875  Public Score 0.86245\n",
    "pipe1_model_hgbc_woe_powertransformer = Pipeline(\n",
    "    steps=[\n",
    "        ('preprocessor', preprocessor_woe_powertransformer),\n",
    "         ('reducedim',  SelectPercentile( f_classif, percentile=95)),\n",
    "        #('dim_red', SelectKBest(f_regression, k=20)),\n",
    "        #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "        #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "        #('PCAreducer', PCA(n_components=0.8)),\n",
    "        ('classifier',model_hgbc)\n",
    "    ]\n",
    ")\n",
    "pipe1_model_hgbc_woe_powertransformer2 = Pipeline(\n",
    "        steps=[\n",
    "            ('preprocessor', preprocessor_woe_powertransformer),\n",
    "            # ('dim_red', SelectKBest(f_regression, k=20)),\n",
    "            #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "             #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "            #('PCAreducer', PCA(n_components=0.8)),\n",
    "            ('classifier',model_hgbc)\n",
    "        ]\n",
    "    )\n",
    "# Private Score 0.86861  Public Score 0.86236\n",
    "pipe1_model_hgbc_woe_powertransformer2 \n",
    "#pipe1_model_hgbc_woe_powertransformer\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adfcebc3",
   "metadata": {
    "papermill": {
     "duration": 0.03818,
     "end_time": "2022-01-10T22:32:19.367388",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.329208",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**2-CatBoost**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b41a65aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:19.456355Z",
     "iopub.status.busy": "2022-01-10T22:32:19.455437Z",
     "iopub.status.idle": "2022-01-10T22:32:19.457673Z",
     "shell.execute_reply": "2022-01-10T22:32:19.458046Z"
    },
    "papermill": {
     "duration": 0.051921,
     "end_time": "2022-01-10T22:32:19.458188",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.406267",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "params_catboost = { \"random_state\": 42,\n",
    "                    \"loss_function\":\"Logloss\",\n",
    "                    \"eval_metric\":\"AUC\",\n",
    "                    #\"task_type\" : \"GPU\",\n",
    "                   'learning_rate': 0.03174251209279996,\n",
    "                   'iterations': 1957,\n",
    "                   'objective': 'CrossEntropy',\n",
    "                   'colsample_bylevel': 0.05457006237382004,\n",
    "                   'depth': 4,\n",
    "                   'boosting_type': 'Ordered',\n",
    "                   'bootstrap_type': 'MVS',\n",
    "                   'verbose':False\n",
    "                  }\n",
    "model_catboost = CatBoostClassifier(\n",
    "                              # use it  only on gpu\n",
    "                            # rsm=np.nan\n",
    "                              #cat_features=cat_features\n",
    "                             **params_catboost)\n",
    "pipe1_model_catboost_woe_powertransformer = Pipeline(\n",
    "    steps=[\n",
    "        ('preprocessor', preprocessor_woe_powertransformer),\n",
    "         ('reducedim',  SelectPercentile( f_classif, percentile=95)),\n",
    "        # ('dim_red', SelectKBest(f_regression, k=20)),\n",
    "        #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "         #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "        #('PCAreducer', PCA(n_components=0.8)),\n",
    "        ('classifier',model_catboost)\n",
    "    ]\n",
    ")\n",
    "# Private Score 0.86903  Public Score 0.86217\n",
    "#pipe1_model_catboost_woe_powertransformer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64f23915",
   "metadata": {
    "papermill": {
     "duration": 0.038425,
     "end_time": "2022-01-10T22:32:19.535027",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.496602",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**-3 XGBR**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3bda5807",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:19.621149Z",
     "iopub.status.busy": "2022-01-10T22:32:19.620285Z",
     "iopub.status.idle": "2022-01-10T22:32:19.623844Z",
     "shell.execute_reply": "2022-01-10T22:32:19.623367Z"
    },
    "papermill": {
     "duration": 0.050397,
     "end_time": "2022-01-10T22:32:19.623970",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.573573",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "params_xgbr = { \"verbosity\": 0,\n",
    "                \"objective\": \"binary:logistic\",\n",
    "                \"eval_metric\": \"auc\",\n",
    "                'random_state': 42,\n",
    "                # regression\n",
    "                #'objective':'reg:squarederror', \n",
    "                # 'eval_metric': 'mae',\n",
    "                #early_stopping_rounds=100 ,\n",
    "                'gpu_id':0, \n",
    "                'predictor':\"gpu_predictor\",\n",
    "                # use exact for small dataset.\n",
    "                #\"tree_method\": \"exact\",\n",
    "                # big data :\n",
    "                 'tree_method': 'gpu_hist',\n",
    "                # defines booster, gblinear for linear functions.\n",
    "               'booster': 'gbtree',\n",
    "               'lambda': 0.05718879852112006,\n",
    "               'alpha': 0.3744455966091751, \n",
    "               'subsample': 0.4600950043573827, \n",
    "               'colsample_bytree': 0.4826795171274075, \n",
    "               'learning_rate': 0.04028462871213848, \n",
    "               'n_estimators': 6258, \n",
    "               'max_depth': 3,\n",
    "               'min_child_weight': 9,\n",
    "               'eta': 1.0243140309668855e-07,\n",
    "               'gamma': 0.0002882867942209271, \n",
    "               'grow_policy': 'lossguide'\n",
    "              }\n",
    "model_xgbr = XGBClassifier(\n",
    "                              # use it  only on gpu\n",
    "                            # rsm=np.nan\n",
    "                              #cat_features=cat_features\n",
    "                             **params_xgbr)\n",
    "pipe1_model_xgbr_woe_powertransformer = Pipeline(\n",
    "    steps=[\n",
    "        ('preprocessor', preprocessor_woe_powertransformer),\n",
    "         ('reducedim',  SelectPercentile( f_classif, percentile=95)),\n",
    "        # ('dim_red', SelectKBest(f_regression, k=20)),\n",
    "        #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "         #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "        #('PCAreducer', PCA(n_components=0.8)),\n",
    "        ('classifier',model_xgbr)\n",
    "    ]\n",
    ")\n",
    "#pipe1_model_xgbr_woe_powertransformer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70fdf1e7",
   "metadata": {
    "papermill": {
     "duration": 0.038492,
     "end_time": "2022-01-10T22:32:19.701556",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.663064",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**4-lgbm:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "267c02db",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:19.788065Z",
     "iopub.status.busy": "2022-01-10T22:32:19.787282Z",
     "iopub.status.idle": "2022-01-10T22:32:19.791124Z",
     "shell.execute_reply": "2022-01-10T22:32:19.790530Z"
    },
    "papermill": {
     "duration": 0.049955,
     "end_time": "2022-01-10T22:32:19.791256",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.741301",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "params_lgbm =  {\n",
    "               \"objective\": \"binary\",\n",
    "                'random_state': 42,\n",
    "                'metric': 'auc',\n",
    "                \"verbosity\": -1,\n",
    "                \"boosting_type\": \"gbdt\",\n",
    "                \"device\": \"gpu\",\n",
    "                #\"early_stopping_rounds\":100,\n",
    "                'learning_rate': 0.005282912028625591, \n",
    "                'n_estimators': 1491,\n",
    "                'lambda_l1': 2.5825901359648176e-05, \n",
    "                'lambda_l2': 4.490094872871969,\n",
    "                'num_leaves': 76,\n",
    "                'feature_fraction': 0.7099515479738928,\n",
    "                'bagging_fraction': 0.8225459856082522,\n",
    "                'bagging_freq': 3,\n",
    "                'min_child_samples': 35\n",
    "               }\n",
    "model_lgbm = lgbm.LGBMClassifier(**params_lgbm,\n",
    "                              # use it  only on gpu\n",
    "                            # rsm=np.nan\n",
    "                              #cat_features=cat_features\n",
    "                             )\n",
    "pipe1_model_lgbm_woe_powertransformer = Pipeline(\n",
    "    steps=[\n",
    "        ('preprocessor', preprocessor_woe_powertransformer),\n",
    "         ('reducedim',  SelectPercentile( f_classif, percentile=95)),\n",
    "        # ('dim_red', SelectKBest(f_regression, k=20)),\n",
    "        #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "         #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "        #('PCAreducer', PCA(n_components=0.8)),\n",
    "        ('classifier',model_lgbm)\n",
    "    ]\n",
    ")\n",
    "#pipe1_model_lgbm_woe_powertransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "62f593f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:19.879839Z",
     "iopub.status.busy": "2022-01-10T22:32:19.875385Z",
     "iopub.status.idle": "2022-01-10T22:32:19.882221Z",
     "shell.execute_reply": "2022-01-10T22:32:19.881780Z"
    },
    "papermill": {
     "duration": 0.049266,
     "end_time": "2022-01-10T22:32:19.882331",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.833065",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lgbm_param2={'learning_rate': 0.0018069834369607075,\n",
    "                                 'max_depth': 8,\n",
    "                                 #'max_features': 4,\n",
    "                                 'min_samples_leaf': 47,\n",
    "                                 #'min_samples_split': 389,\n",
    "                                 'subsample': 0.8573598985000007,\n",
    "                                 #'n_iter_no_change': 300,\n",
    "                                 'n_estimators': 5000,\n",
    "                                 'verbose': -1,\n",
    "                                 'random_state': 42,\n",
    "                                 'metric': 'auc',\n",
    "                                \"device_type\" : \"gpu\",\n",
    "                                'boosting_type': 'gbdt',\n",
    "                                'tree_method': \"gpu_hist\"\n",
    "                               }\n",
    "\n",
    "model_lgbm2 = lgbm.LGBMClassifier(**lgbm_param2,\n",
    "                              # use it  only on gpu\n",
    "                            # rsm=np.nan\n",
    "                              #cat_features=cat_features\n",
    "                             )\n",
    "pipe1_model_lgbm_woe_powertransformer2 = Pipeline(\n",
    "    steps=[\n",
    "        ('preprocessor', preprocessor_woe_powertransformer),\n",
    "         ('reducedim',  SelectPercentile( f_classif, percentile=95)),\n",
    "        # ('dim_red', SelectKBest(f_regression, k=20)),\n",
    "        #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "         #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "        #('PCAreducer', PCA(n_components=0.8)),\n",
    "        ('classifier',model_lgbm2)\n",
    "    ]\n",
    ")\n",
    "#pipe1_model_lgbm_woe_powertransformer2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9716d447",
   "metadata": {
    "papermill": {
     "duration": 0.04184,
     "end_time": "2022-01-10T22:32:19.962974",
     "exception": false,
     "start_time": "2022-01-10T22:32:19.921134",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Best Pipes : \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e579c06a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:20.060249Z",
     "iopub.status.busy": "2022-01-10T22:32:20.049674Z",
     "iopub.status.idle": "2022-01-10T22:32:20.062667Z",
     "shell.execute_reply": "2022-01-10T22:32:20.062193Z"
    },
    "papermill": {
     "duration": 0.060891,
     "end_time": "2022-01-10T22:32:20.062788",
     "exception": false,
     "start_time": "2022-01-10T22:32:20.001897",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pipe1_lgbm_woe_powertransformer93 = Pipeline(\n",
    "        steps=[\n",
    "            ('preprocessor', preprocessor_woe_powertransformer),\n",
    "            ('reducedim',  SelectPercentile( f_classif, percentile=93)),\n",
    "            ('classifier',\n",
    "             lgbm.LGBMClassifier( \n",
    "                #n_jobs=-1,\n",
    "                                             \n",
    "                                               **lgbm_param2))\n",
    "        ]\n",
    "    )\n",
    "\n",
    "pipe1_lgbm_woe_powertransformer = Pipeline(\n",
    "                                steps=[\n",
    "                                    ('preprocessor', preprocessor_woe_powertransformer),\n",
    "                                    ('reducedim',  SelectPercentile( f_classif, percentile=98)),\n",
    "                                    ('classifier',\n",
    "                                     lgbm.LGBMClassifier( \n",
    "                                        #n_jobs=-1,\n",
    "                                                                    \n",
    "                                                                       **lgbm_param2))\n",
    "                                ]\n",
    "                                )\n",
    "pipe1_lgbm_woe_powertransformer90 = Pipeline(\n",
    "        steps=[\n",
    "            ('preprocessor', preprocessor_woe_powertransformer),\n",
    "            ('reducedim',  SelectPercentile( f_classif, percentile=90)),\n",
    "            ('classifier',\n",
    "             lgbm.LGBMClassifier( \n",
    "                #n_jobs=-1,\n",
    "                                           \n",
    "                                               **lgbm_param2))\n",
    "        ]\n",
    "    )\n",
    "pipe1_lgbm_woe_powertransformer95 = Pipeline(\n",
    "        steps=[\n",
    "            ('preprocessor', preprocessor_woe_powertransformer),\n",
    "            ('reducedim',  SelectPercentile( f_classif, percentile=95)),\n",
    "            ('classifier',\n",
    "             lgbm.LGBMClassifier( \n",
    "                #n_jobs=-1,\n",
    "                                           \n",
    "                                               **lgbm_param2))\n",
    "        ]\n",
    "    )\n",
    "preprocessor_woe_powertransformer100 = ColumnTransformer(\n",
    "                transformers=[\n",
    "                    ('numerical', numeric_transformer, num_columns),\n",
    "                   # ('categorical', categorical_transformer, cat_columns)\n",
    "                ])\n",
    "pipe_lgbm_woe_powertransformer100 = Pipeline(\n",
    "    steps=[\n",
    "        ('preprocessor', preprocessor_woe_powertransformer100),\n",
    "        # ('dim_red', SelectKBest(f_regression, k=20)),\n",
    "        #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "         #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "        #('PCAreducer', PCA(n_components=0.8)),\n",
    "        ('classifier',\n",
    "         lgbm.LGBMClassifier( \n",
    "            #n_jobs=-1,\n",
    "                                         \n",
    "                                           **lgbm_param2))\n",
    "    ]\n",
    "    ) \n",
    "## Pipe_12\n",
    "#Define vnum pipeline\n",
    "numeric_transformer_12 = Pipeline(\n",
    "                                steps=[\n",
    "                                ('imputer', SimpleImputer(strategy='median'\n",
    "                                                          ,add_indicator=True)),\n",
    "                                #('general_features',FeaturesEngineer()),\n",
    "                                ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                                # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                                #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                                ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                                ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                                #('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                                ]\n",
    "                                )\n",
    "\n",
    "preprocessor_woe_powertransformer_12 = ColumnTransformer(\n",
    "                transformers=[\n",
    "                    ('numerical', numeric_transformer_12, num_columns),\n",
    "                   # ('categorical', categorical_transformer, cat_columns)\n",
    "                ])\n",
    "pipe1_lgbm_woe_powertransformer_12 = Pipeline(\n",
    "        steps=[\n",
    "            ('preprocessor', preprocessor_woe_powertransformer_12),\n",
    "            ('classifier',\n",
    "             lgbm.LGBMClassifier( \n",
    "                #n_jobs=-1,\n",
    "                                               \n",
    "                                               **lgbm_param2))\n",
    "        ]\n",
    "    )\n",
    "# Pipe 2: \n",
    "# Cat pipeline\n",
    "categorical_transformer2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "                        #('sparse_features', SparseInteractions(degree=2)),\n",
    "                        #('reducedim',  SelectPercentile( mutual_info_classif, percentile=90))\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            #('general_features',FeaturesEngineer()),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            #('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer2 = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer2, num_columns),\n",
    "                ('categorical', categorical_transformer2, cat_columns)\n",
    "            ])\n",
    "pipe2_lgbm_woe_powertransformer = Pipeline(\n",
    "        steps=[\n",
    "            ('preprocessor', preprocessor_woe_powertransformer2),\n",
    "            ('classifier',\n",
    "             lgbm.LGBMClassifier( \n",
    "                #n_jobs=-1,\n",
    "                                              \n",
    "                                               **lgbm_param2))\n",
    "        ]\n",
    "    )\n",
    "pipe2_model_catboost_woe_powertransformer = Pipeline(\n",
    "    steps=[\n",
    "        ('preprocessor', preprocessor_woe_powertransformer2),\n",
    "         ('reducedim',  SelectPercentile( f_classif, percentile=95)),\n",
    "        # ('dim_red', SelectKBest(f_regression, k=20)),\n",
    "        #('Sparseinteractions', SparseInteractions(degree=2)),\n",
    "         #('PolynomialFeatures', PolynomialFeatures(degree=2)),\n",
    "        #('PCAreducer', PCA(n_components=0.8)),\n",
    "        ('classifier',model_catboost)\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0115ec1e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:20.146317Z",
     "iopub.status.busy": "2022-01-10T22:32:20.145510Z",
     "iopub.status.idle": "2022-01-10T22:32:20.147952Z",
     "shell.execute_reply": "2022-01-10T22:32:20.147508Z"
    },
    "papermill": {
     "duration": 0.046127,
     "end_time": "2022-01-10T22:32:20.148061",
     "exception": false,
     "start_time": "2022-01-10T22:32:20.101934",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pipe 1\n",
    "# Private Score 0.86914 Public Score 0.86245\n",
    "pipe1_lgbm_woe_powertransformer95 \n",
    "# Private Score 0.86911 Public Score 0.86230\n",
    "pipe_lgbm_woe_powertransformer100  \n",
    "# Private Score 0.86903  Public Score 0.86217\n",
    "pipe1_model_catboost_woe_powertransformer\n",
    "#Private Score 0.86875  Public Score 0.86245\n",
    "pipe1_model_hgbc_woe_powertransformer\n",
    "## Pipe_12\n",
    "#  Private Score 0.86895  Public Score 0.86231\n",
    "pipe1_lgbm_woe_powertransformer_12\n",
    "# Private Score 0.86898 Public Score 0.86230\n",
    "pipe2_lgbm_woe_powertransformer\n",
    "# Private Score 0.86903  Public Score 0.86215\n",
    "pipe2_model_catboost_woe_powertransformer  \n",
    "best_pipes=[pipe1_lgbm_woe_powertransformer95,\n",
    "            pipe1_lgbm_woe_powertransformer_12,\n",
    "            pipe1_model_catboost_woe_powertransformer,\n",
    "            #pipe1_model_hgbc_woe_powertransformer,\n",
    "            pipe2_lgbm_woe_powertransformer,\n",
    "            pipe2_model_catboost_woe_powertransformer\n",
    "           ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "191403b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:20.234122Z",
     "iopub.status.busy": "2022-01-10T22:32:20.233585Z",
     "iopub.status.idle": "2022-01-10T22:32:29.795712Z",
     "shell.execute_reply": "2022-01-10T22:32:29.796139Z"
    },
    "papermill": {
     "duration": 9.609221,
     "end_time": "2022-01-10T22:32:29.796288",
     "exception": false,
     "start_time": "2022-01-10T22:32:20.187067",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"78684d81-c769-452e-8fac-54a33aa36b8a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"78684d81-c769-452e-8fac-54a33aa36b8a\">StackingCVClassifier</label><div class=\"sk-toggleable__content\"><pre>StackingCVClassifier(classifiers=[Pipeline(steps=[('preprocessor',\n",
       "                                                   ColumnTransformer(transformers=[('numerical',\n",
       "                                                                                    Pipeline(steps=[('imputer',\n",
       "                                                                                                     SimpleImputer(add_indicator=True,\n",
       "                                                                                                                   strategy='median')),\n",
       "                                                                                                    ('scaler',\n",
       "                                                                                                     PowerTransformer()),\n",
       "                                                                                                    ('kmeans',\n",
       "                                                                                                     MiniKmeansTransformerEncoder()),\n",
       "                                                                                                    ('polynominal_features',\n",
       "                                                                                                     PolynomialFeatures())]),\n",
       "                                                                                    Index(['RevolvingUtilizationOfUnsecured...\n",
       "                                                   learning_rate=0.04028462871213848,\n",
       "                                                   max_delta_step=None,\n",
       "                                                   max_depth=3,\n",
       "                                                   min_child_weight=9,\n",
       "                                                   missing=nan,\n",
       "                                                   monotone_constraints=None,\n",
       "                                                   n_estimators=6258,\n",
       "                                                   n_jobs=None,\n",
       "                                                   num_parallel_tree=None,\n",
       "                                                   predictor='gpu_predictor',\n",
       "                                                   random_state=42,\n",
       "                                                   reg_alpha=None,\n",
       "                                                   reg_lambda=None,\n",
       "                                                   scale_pos_weight=None,\n",
       "                                                   subsample=0.4600950043573827, ...),\n",
       "                     random_state=42, use_features_in_secondary=True,\n",
       "                     use_probas=True)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"0da14564-7a3f-4577-91a1-a5adf236f09a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"0da14564-7a3f-4577-91a1-a5adf236f09a\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(alpha=0.3744455966091751, base_score=None, booster='gbtree',\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=0.4826795171274075, enable_categorical=False,\n",
       "              eta=1.0243140309668855e-07, eval_metric='auc',\n",
       "              gamma=0.0002882867942209271, gpu_id=0, grow_policy='lossguide',\n",
       "              importance_type=None, interaction_constraints=None,\n",
       "              lambda=0.05718879852112006, learning_rate=0.04028462871213848,\n",
       "              max_delta_step=None, max_depth=3, min_child_weight=9, missing=nan,\n",
       "              monotone_constraints=None, n_estimators=6258, n_jobs=None,\n",
       "              num_parallel_tree=None, predictor='gpu_predictor',\n",
       "              random_state=42, reg_alpha=None, reg_lambda=None,\n",
       "              scale_pos_weight=None, subsample=0.4600950043573827, ...)</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a543edff-3374-48d2-a1c2-f4e09111d9eb\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a543edff-3374-48d2-a1c2-f4e09111d9eb\">preprocessor: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"df304625-ff4e-4b60-a1e2-8abb8c94fa9f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"df304625-ff4e-4b60-a1e2-8abb8c94fa9f\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"08057629-316c-4b3a-b878-aed69a266432\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"08057629-316c-4b3a-b878-aed69a266432\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"0ce3b5ab-a4af-418c-a26c-773ab937b7e9\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"0ce3b5ab-a4af-418c-a26c-773ab937b7e9\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"3535fc63-e6fa-40bb-a888-ecf31ef189b4\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"3535fc63-e6fa-40bb-a888-ecf31ef189b4\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"3309287e-0b68-4736-95e1-e00e8a1b71af\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"3309287e-0b68-4736-95e1-e00e8a1b71af\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"75763af7-ac0b-49bf-a90d-f59358a319ab\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"75763af7-ac0b-49bf-a90d-f59358a319ab\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"e0f3dde0-5d99-4b15-9078-3a5d4a942791\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"e0f3dde0-5d99-4b15-9078-3a5d4a942791\">SelectPercentile</label><div class=\"sk-toggleable__content\"><pre>SelectPercentile(percentile=95)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a35c5002-02ee-4ed7-a7c1-188c3f18cc04\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a35c5002-02ee-4ed7-a7c1-188c3f18cc04\">LGBMClassifier</label><div class=\"sk-toggleable__content\"><pre>LGBMClassifier(device_type='gpu', learning_rate=0.0018069834369607075,\n",
       "               max_depth=8, metric='auc', min_samples_leaf=47,\n",
       "               n_estimators=5000, random_state=42, subsample=0.8573598985000007,\n",
       "               tree_method='gpu_hist', verbose=-1)</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"2d78b67d-4423-4b3b-9e89-61a4fb665074\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"2d78b67d-4423-4b3b-9e89-61a4fb665074\">preprocessor: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"cbec5b78-9b90-42cd-85ff-2f019784707c\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"cbec5b78-9b90-42cd-85ff-2f019784707c\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ab8dd2f7-1f42-4099-afad-668824aebe8a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ab8dd2f7-1f42-4099-afad-668824aebe8a\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a5707a4d-3669-4514-83b4-fa2f1402cc17\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a5707a4d-3669-4514-83b4-fa2f1402cc17\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"cc6464b4-8b6a-47b0-92d9-709577d36a28\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"cc6464b4-8b6a-47b0-92d9-709577d36a28\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"f98fba58-3d40-4e85-b6c0-efe9bbca4be6\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"f98fba58-3d40-4e85-b6c0-efe9bbca4be6\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"2fa06df9-30a1-4eb4-9731-6f4e3dd43077\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"2fa06df9-30a1-4eb4-9731-6f4e3dd43077\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"322213e0-3064-468d-a9f4-6f6c85876cd3\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"322213e0-3064-468d-a9f4-6f6c85876cd3\">LGBMClassifier</label><div class=\"sk-toggleable__content\"><pre>LGBMClassifier(device_type='gpu', learning_rate=0.0018069834369607075,\n",
       "               max_depth=8, metric='auc', min_samples_leaf=47,\n",
       "               n_estimators=5000, random_state=42, subsample=0.8573598985000007,\n",
       "               tree_method='gpu_hist', verbose=-1)</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a50ac205-4b28-40f1-8a59-77113eea05e1\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a50ac205-4b28-40f1-8a59-77113eea05e1\">preprocessor: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"84cadb82-c264-455c-80b2-6d9c31080361\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"84cadb82-c264-455c-80b2-6d9c31080361\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"f102ab4d-9009-4406-8c40-dff906ce8088\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"f102ab4d-9009-4406-8c40-dff906ce8088\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a6a5051b-405e-4c90-a1b0-ed9cf96eda21\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a6a5051b-405e-4c90-a1b0-ed9cf96eda21\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a97bc993-73ef-4c5e-9f40-8ccff639573f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a97bc993-73ef-4c5e-9f40-8ccff639573f\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"8dbf9138-bfdc-4c2d-a404-739c08c07138\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"8dbf9138-bfdc-4c2d-a404-739c08c07138\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"48403fd9-6c1c-4d3c-a9e6-ae4e8e6e1730\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"48403fd9-6c1c-4d3c-a9e6-ae4e8e6e1730\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c2102b66-2c26-47b4-b219-b35122d4eeb0\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c2102b66-2c26-47b4-b219-b35122d4eeb0\">SelectPercentile</label><div class=\"sk-toggleable__content\"><pre>SelectPercentile(percentile=95)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ad679a27-cb6f-437f-bd01-4bc0fdb016b4\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ad679a27-cb6f-437f-bd01-4bc0fdb016b4\">CatBoostClassifier</label><div class=\"sk-toggleable__content\"><pre><catboost.core.CatBoostClassifier object at 0x7f8cc0935610></pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"de654857-0b7f-4b6b-86c2-9e88491bc966\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"de654857-0b7f-4b6b-86c2-9e88491bc966\">preprocessor: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfT...\n",
       "                                                 ('encoder', WOEEncoder())]),\n",
       "                                 Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"b2e5f099-408c-4262-8b8e-f9bfbddb301d\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"b2e5f099-408c-4262-8b8e-f9bfbddb301d\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"573165df-52a1-4fab-ac03-a15e80d97deb\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"573165df-52a1-4fab-ac03-a15e80d97deb\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c78ca42c-a768-4a1a-a582-3ed4cecbd275\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c78ca42c-a768-4a1a-a582-3ed4cecbd275\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"26f40443-f7e3-494f-8b90-35b9ab002069\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"26f40443-f7e3-494f-8b90-35b9ab002069\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"dcd2699d-acce-44fa-90b9-8927d7590e57\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"dcd2699d-acce-44fa-90b9-8927d7590e57\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"80c42574-f254-4225-b7a8-948579e5e4e1\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"80c42574-f254-4225-b7a8-948579e5e4e1\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ba060be2-d25b-4482-8d42-a2a095dfe7a5\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ba060be2-d25b-4482-8d42-a2a095dfe7a5\">categorical</label><div class=\"sk-toggleable__content\"><pre>Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"6455b1e7-e5e9-422d-be85-e884fe7e706c\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"6455b1e7-e5e9-422d-be85-e884fe7e706c\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, fill_value='missing',\n",
       "              strategy='most_frequent')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"d45f0f9d-090a-481c-864e-88bbb2ad7ee0\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"d45f0f9d-090a-481c-864e-88bbb2ad7ee0\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a3aa3198-eeab-4378-b59e-b1c722f793f5\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a3aa3198-eeab-4378-b59e-b1c722f793f5\">LGBMClassifier</label><div class=\"sk-toggleable__content\"><pre>LGBMClassifier(device_type='gpu', learning_rate=0.0018069834369607075,\n",
       "               max_depth=8, metric='auc', min_samples_leaf=47,\n",
       "               n_estimators=5000, random_state=42, subsample=0.8573598985000007,\n",
       "               tree_method='gpu_hist', verbose=-1)</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"f0f04a4d-4230-4bed-b93b-c0d92f4a32bd\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"f0f04a4d-4230-4bed-b93b-c0d92f4a32bd\">preprocessor: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfT...\n",
       "                                                 ('encoder', WOEEncoder())]),\n",
       "                                 Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"93e653cd-0b93-4522-a7e0-812f7609971b\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"93e653cd-0b93-4522-a7e0-812f7609971b\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"079476eb-4fe4-4b21-a1fc-6abf842a681a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"079476eb-4fe4-4b21-a1fc-6abf842a681a\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ce4aa24a-3acb-4cc1-89fc-a997d0dbc053\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ce4aa24a-3acb-4cc1-89fc-a997d0dbc053\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a6132dc6-f68e-4491-9760-48ad4521b8f4\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a6132dc6-f68e-4491-9760-48ad4521b8f4\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"cf69d3bb-1836-42f4-a17e-ab55ea5bfa33\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"cf69d3bb-1836-42f4-a17e-ab55ea5bfa33\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"871a5458-732c-452e-b738-b7fef6c205a0\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"871a5458-732c-452e-b738-b7fef6c205a0\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"241a5134-283e-477d-b921-a6baa5bfcd93\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"241a5134-283e-477d-b921-a6baa5bfcd93\">categorical</label><div class=\"sk-toggleable__content\"><pre>Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"3a4ef1fb-2003-4313-ac54-5141bbe9f7f3\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"3a4ef1fb-2003-4313-ac54-5141bbe9f7f3\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, fill_value='missing',\n",
       "              strategy='most_frequent')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"f4ccc87d-53d8-4f9d-ba09-619cf842b393\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"f4ccc87d-53d8-4f9d-ba09-619cf842b393\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"5cae599d-39d4-4aa5-998e-f84c72cf3106\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"5cae599d-39d4-4aa5-998e-f84c72cf3106\">SelectPercentile</label><div class=\"sk-toggleable__content\"><pre>SelectPercentile(percentile=95)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"04e70c2d-b56c-48e3-b3d9-15c5d8269ea1\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"04e70c2d-b56c-48e3-b3d9-15c5d8269ea1\">CatBoostClassifier</label><div class=\"sk-toggleable__content\"><pre><catboost.core.CatBoostClassifier object at 0x7f8cc0935610></pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "StackingCVClassifier(classifiers=[Pipeline(steps=[('preprocessor',\n",
       "                                                   ColumnTransformer(transformers=[('numerical',\n",
       "                                                                                    Pipeline(steps=[('imputer',\n",
       "                                                                                                     SimpleImputer(add_indicator=True,\n",
       "                                                                                                                   strategy='median')),\n",
       "                                                                                                    ('scaler',\n",
       "                                                                                                     PowerTransformer()),\n",
       "                                                                                                    ('kmeans',\n",
       "                                                                                                     MiniKmeansTransformerEncoder()),\n",
       "                                                                                                    ('polynominal_features',\n",
       "                                                                                                     PolynomialFeatures())]),\n",
       "                                                                                    Index(['RevolvingUtilizationOfUnsecured...\n",
       "                                                   learning_rate=0.04028462871213848,\n",
       "                                                   max_delta_step=None,\n",
       "                                                   max_depth=3,\n",
       "                                                   min_child_weight=9,\n",
       "                                                   missing=nan,\n",
       "                                                   monotone_constraints=None,\n",
       "                                                   n_estimators=6258,\n",
       "                                                   n_jobs=None,\n",
       "                                                   num_parallel_tree=None,\n",
       "                                                   predictor='gpu_predictor',\n",
       "                                                   random_state=42,\n",
       "                                                   reg_alpha=None,\n",
       "                                                   reg_lambda=None,\n",
       "                                                   scale_pos_weight=None,\n",
       "                                                   subsample=0.4600950043573827, ...),\n",
       "                     random_state=42, use_features_in_secondary=True,\n",
       "                     use_probas=True)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##### from mlxtend.plotting import plot_confusion_matrix\n",
    "from mlxtend.classifier import StackingCVClassifier\n",
    "\n",
    "mlxtend_sclf1 = StackingCVClassifier(classifiers=best_pipes,use_probas=False,\n",
    "                            meta_classifier=model_xgbr,cv=10,stratify=True,\n",
    "                            random_state=42)\n",
    "\n",
    "mlxtend_sclf2 = StackingCVClassifier(classifiers=best_pipes,use_probas=True,\n",
    "                            meta_classifier=model_xgbr,cv=10,stratify=True,\n",
    "                            random_state=42)\n",
    "\n",
    "mlxtend_sclf3 = StackingCVClassifier(classifiers=best_pipes,use_probas=True,\n",
    "                              use_features_in_secondary=True ,\n",
    "                              meta_classifier=model_xgbr,cv=10,stratify=True,\n",
    "                            random_state=42)\n",
    "mlxtend_sclf32 = StackingCVClassifier(classifiers=best_pipes,use_probas=True,\n",
    "                              use_features_in_secondary=True ,\n",
    "                              meta_classifier=LogisticRegression(),cv=10,stratify=True,\n",
    "                            random_state=42)\n",
    "mlxtend_sclf4_pipe = StackingCVClassifier(classifiers=best_pipes,use_probas=True,\n",
    "                              use_features_in_secondary=True ,\n",
    "                              meta_classifier=model_xgbr,cv=10,stratify=True,\n",
    "                            random_state=42)\n",
    "mlxtend_sclf3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "965e7885",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:29.880766Z",
     "iopub.status.busy": "2022-01-10T22:32:29.880180Z",
     "iopub.status.idle": "2022-01-10T22:32:29.883816Z",
     "shell.execute_reply": "2022-01-10T22:32:29.883255Z"
    },
    "papermill": {
     "duration": 0.047315,
     "end_time": "2022-01-10T22:32:29.883938",
     "exception": false,
     "start_time": "2022-01-10T22:32:29.836623",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#X=X.iloc[0:10000,:].copy()\n",
    "#y=y.iloc[0:10000].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a57de0ed",
   "metadata": {
    "papermill": {
     "duration": 0.040371,
     "end_time": "2022-01-10T22:32:29.964033",
     "exception": false,
     "start_time": "2022-01-10T22:32:29.923662",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Tuning Stacking Process: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "95739b55",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:30.048665Z",
     "iopub.status.busy": "2022-01-10T22:32:30.047846Z",
     "iopub.status.idle": "2022-01-10T22:32:30.154468Z",
     "shell.execute_reply": "2022-01-10T22:32:30.155026Z"
    },
    "papermill": {
     "duration": 0.150977,
     "end_time": "2022-01-10T22:32:30.155200",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.004223",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "#                                  Get data                                 #\n",
    "###############################################################################\n",
    "X_train,X_test, y_train,  y_test = train_test_split(X, y,stratify=y, \n",
    "                                                       test_size=0.2,\n",
    "                                                        random_state=42,\n",
    "                                                        shuffle=True)\n",
    "##############################################################################\n",
    "#                               Good Ol' Classifiers                        #\n",
    "###############################################################################\n",
    "best_pipes\n",
    "###############################################################################\n",
    "#                             Stacking Classifier                          #\n",
    "###############################################################################\n",
    "# Initializing the StackingCV classifier\n",
    "mlxtend_sclf2 = StackingCVClassifier(classifiers=best_pipes,use_probas=True,\n",
    "                            meta_classifier=LogisticRegression(),cv=4,stratify=True,\n",
    "                            random_state=42)\n",
    "\n",
    "###############################################################################\n",
    "#                        Putting classifiers in a dictionary                #\n",
    "###############################################################################\n",
    "\n",
    "# Create list to store classifiers\n",
    "classifiers = {\"pipe1\":pipe1_lgbm_woe_powertransformer95,\n",
    "            #\"pipe2\":pipe_lgbm_woe_powertransformer100,\n",
    "            \"pipe2\": pipe1_lgbm_woe_powertransformer_12,\n",
    "           # \"pipe3\":pipe1_model_catboost_woe_powertransformer,\n",
    "           # \"pipe5\":pipe1_model_hgbc_woe_powertransformer,\n",
    "            \"pipe3\":pipe2_lgbm_woe_powertransformer,\n",
    "            \"pipe4\":pipe2_model_catboost_woe_powertransformer\n",
    "           }\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3389a02b",
   "metadata": {
    "papermill": {
     "duration": 0.040179,
     "end_time": "2022-01-10T22:32:30.235904",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.195725",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "    ###############################################################################\n",
    "    #                                Train classifiers                          #\n",
    "    ###############################################################################\n",
    "    # Train classifiers\n",
    "    for key in classifiers:\n",
    "        # Get classifier\n",
    "        classifier = classifiers[key]\n",
    "\n",
    "        # Fit classifier\n",
    "        classifier.fit(X, y)\n",
    "\n",
    "        # Save fitted classifier\n",
    "        classifiers[key] = classifier\n",
    "\n",
    "\n",
    "    ###############################################################################\n",
    "    #                               Making predictions                          #\n",
    "    ###############################################################################\n",
    "    # Get results\n",
    "    results = pd.DataFrame()\n",
    "    for key in classifiers:\n",
    "        # Make prediction on test set\n",
    "        y_pred = classifiers[key].predict_proba(X_test)[:,1]\n",
    "\n",
    "        # Save results in pandas dataframe object\n",
    "        results[f\"{key}\"] = y_pred\n",
    "\n",
    "    # Add the test set to the results object\n",
    "    results[\"Target\"] = y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2b131b4",
   "metadata": {
    "papermill": {
     "duration": 0.039658,
     "end_time": "2022-01-10T22:32:30.315771",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.276113",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    " # Visualzing results   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b805762",
   "metadata": {
    "papermill": {
     "duration": 0.039842,
     "end_time": "2022-01-10T22:32:30.395880",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.356038",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "    ###############################################################################\n",
    "    #                               Visualzing results                         #\n",
    "    ###############################################################################\n",
    "    # Probability Distributions Figure\n",
    "    # Set graph style\n",
    "    sns.set(font_scale = 1)\n",
    "    sns.set_style({\"axes.facecolor\": \"1.0\", \"axes.edgecolor\": \"0.85\", \"grid.color\": \"0.85\",\n",
    "                   \"grid.linestyle\": \"-\", 'axes.labelcolor': '0.4', \"xtick.color\": \"0.4\",\n",
    "                   'ytick.color': '0.4'})\n",
    "\n",
    "    # Plot\n",
    "    f, ax = plt.subplots(figsize=(13, 4), nrows=1, ncols = 4)\n",
    "\n",
    "    for key, counter in zip(classifiers, range(4)):\n",
    "        # Get predictions\n",
    "        y_pred = results[key]\n",
    "\n",
    "        # Get AUCm\n",
    "        auc = roc_auc_score(y_test, y_pred)\n",
    "        textstr = f\"AUC: {auc:.3f}\"\n",
    "\n",
    "        # Plot false distribution\n",
    "        false_pred = results[results[\"Target\"] == 0]\n",
    "        sns.distplot(false_pred[key], hist=True, kde=False, \n",
    "                     bins=int(25), color = 'red',\n",
    "                     hist_kws={'edgecolor':'black'}, ax = ax[counter])\n",
    "\n",
    "        # Plot true distribution\n",
    "        true_pred = results[results[\"Target\"] == 1]\n",
    "        sns.distplot(results[key], hist=True, kde=False, \n",
    "                     bins=int(25), color = 'green',\n",
    "                     hist_kws={'edgecolor':'black'}, ax = ax[counter])\n",
    "\n",
    "\n",
    "        # These are matplotlib.patch.Patch properties\n",
    "        props = dict(boxstyle='round', facecolor='white', alpha=0.5)\n",
    "\n",
    "        # Place a text box in upper left in axes coords\n",
    "        ax[counter].text(0.05, 0.95, textstr, transform=ax[counter].transAxes, fontsize=14,\n",
    "                        verticalalignment = \"top\", bbox=props)\n",
    "\n",
    "        # Set axis limits and labels\n",
    "        ax[counter].set_title(f\"{key} Distribution\")\n",
    "        ax[counter].set_xlim(0,1)\n",
    "        ax[counter].set_xlabel(\"Probability\")\n",
    "\n",
    "    # Tight layout\n",
    "    plt.tight_layout()\n",
    "    # Save Figure\n",
    "    plt.savefig(\"Probability Distribution for each Classifier.png\", dpi = 1080)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1345dd4",
   "metadata": {
    "papermill": {
     "duration": 0.039782,
     "end_time": "2022-01-10T22:32:30.475420",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.435638",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Tuning the Meta-Classifier  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4424c394",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:30.565053Z",
     "iopub.status.busy": "2022-01-10T22:32:30.564127Z",
     "iopub.status.idle": "2022-01-10T22:32:30.566043Z",
     "shell.execute_reply": "2022-01-10T22:32:30.566505Z"
    },
    "papermill": {
     "duration": 0.050753,
     "end_time": "2022-01-10T22:32:30.566661",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.515908",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "#                         Tuning the Meta-Classifier                      #\n",
    "###############################################################################\n",
    "# Define parameter grid \n",
    "#paramssvc = {\"meta_classifier__kernel\": [\"linear\", \"rbf\", \"poly\"],\n",
    " #         \"meta_classifier__C\": [1, 2],\n",
    "  #        \"meta_classifier__degree\": [3, 4, 5],\n",
    "   #     \n",
    "params = {\"meta_classifier__random_state\": [42],\n",
    "         \"meta_classifier__solver\": ['newton-cg'],\n",
    "         \"meta_classifier__max_iter\": [10],\n",
    "        #\"meta_classifier__C\": np.logspace(-4, 4, 10),\n",
    "        #\"meta_classifier__C\": np.logspace(-4, 4, 10),\n",
    "         \"meta_classifier__C\": [.001],\n",
    "         #\"meta_classifier__penalty\":  ['l2', 'l1']\n",
    "         \"meta_classifier__penalty\":  ['l2']\n",
    "         }\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3f055e8",
   "metadata": {
    "papermill": {
     "duration": 0.041774,
     "end_time": "2022-01-10T22:32:30.650205",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.608431",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "    # Initialize GridSearchCV\n",
    "    grid = GridSearchCV(estimator = mlxtend_sclf2, \n",
    "                        param_grid = params, \n",
    "                        cv = 3,\n",
    "                        scoring = \"roc_auc\",\n",
    "                        verbose = 0,\n",
    "                        n_jobs = -1)\n",
    "\n",
    "    # Fit GridSearchCV\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    # Making prediction on test set\n",
    "    y_pred = grid.predict_proba(X_test)[:,1]\n",
    "\n",
    "    # Getting AUC\n",
    "    auc = roc_auc_score(y_test, y_pred)\n",
    "\n",
    "    # Print results\n",
    "    print(f\"The AUC of the tuned Stacking classifier is {auc:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0c2975ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:30.738744Z",
     "iopub.status.busy": "2022-01-10T22:32:30.737861Z",
     "iopub.status.idle": "2022-01-10T22:32:30.739630Z",
     "shell.execute_reply": "2022-01-10T22:32:30.740060Z"
    },
    "papermill": {
     "duration": 0.04866,
     "end_time": "2022-01-10T22:32:30.740191",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.691531",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#grid.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a1a100b",
   "metadata": {
    "papermill": {
     "duration": 0.040061,
     "end_time": "2022-01-10T22:32:30.821007",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.780946",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Stacking Different Combinations of Classifiers  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "62d57f44",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-10T22:32:30.912490Z",
     "iopub.status.busy": "2022-01-10T22:32:30.911689Z",
     "iopub.status.idle": "2022-01-11T03:05:25.246849Z",
     "shell.execute_reply": "2022-01-11T03:05:25.247265Z"
    },
    "papermill": {
     "duration": 16374.385398,
     "end_time": "2022-01-11T03:05:25.247434",
     "exception": false,
     "start_time": "2022-01-10T22:32:30.862036",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.9min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe1', 'pipe2'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.6min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe1', 'pipe3'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe1', 'pipe4'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.8min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe2', 'pipe3'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 11.7min finished\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe2', 'pipe4'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 12.1min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 12.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe3', 'pipe4'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 18.2min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 18.2min finished\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe1', 'pipe2', 'pipe3'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 17.8min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 17.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe1', 'pipe2', 'pipe4'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 17.4min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 17.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe1', 'pipe3', 'pipe4'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 17.4min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 17.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "AUC of stack ('pipe2', 'pipe3', 'pipe4'): 0.867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "Custom logger is already specified. Specify more than one logger at same time is not thread safe./opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 24.8min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed: 24.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "Custom logger is already specified. Specify more than one logger at same time is not thread safe./opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n",
      "[LightGBM] [Warning] min_data_in_leaf is set with min_child_samples=20, will be overridden by min_samples_leaf=47. Current value: min_data_in_leaf=47\n",
      "[LightGBM] [Warning] Unknown parameter: tree_method\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [0] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/preprocessing/_data.py:2960: RuntimeWarning: overflow encountered in power\n",
      "  out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:205: RuntimeWarning: overflow encountered in multiply\n",
      "  x = um.multiply(x, x, out=x)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: UserWarning: Features [ 0 71 73 75] are constant.\n",
      "  UserWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f = msb / msw\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC of stack ('pipe1', 'pipe2', 'pipe3', 'pipe4'): 0.868\n"
     ]
    }
   ],
   "source": [
    "###############################################################################\n",
    "#               Stacking Different Combinations of Classifiers            #\n",
    "###############################################################################\n",
    "# Classifier labels\n",
    "classifier_labels = [\"pipe1\",\"pipe2\",\"pipe3\",\"pipe4\"]\n",
    "\n",
    "# Get all unique combinations of classifier with a set size greater than or equal to 2\n",
    "combo_classifiers = []\n",
    "for ii in range(2, len(classifier_labels)+1):\n",
    "    for subset in itertools.combinations(classifier_labels, ii):\n",
    "        combo_classifiers.append(subset)\n",
    "# Stack, tune, and evaluate stack of classifiers\n",
    "for combo in combo_classifiers:\n",
    "    # Get labels of classifier to create a stack\n",
    "    labels = list(combo)\n",
    "     \n",
    "    # Get classifiers\n",
    "    classifier_combo = []\n",
    "    for ii in range(len(labels)):\n",
    "        label = classifier_labels[ii]\n",
    "        classifier = classifiers[label]\n",
    "        classifier_combo.append(classifier)\n",
    "         \n",
    "    # Initializing the StackingCV classifier\n",
    "    sclf = StackingCVClassifier(classifiers = classifier_combo,\n",
    "                                shuffle = False,\n",
    "                                use_probas = True,\n",
    "                                cv = 2,\n",
    "                                meta_classifier = LogisticRegression(),\n",
    "                                n_jobs = -1)\n",
    "    \n",
    "    # Initialize GridSearchCV\n",
    "    grid = GridSearchCV(estimator = sclf, \n",
    "                        param_grid = params, \n",
    "                        cv = 2,\n",
    "                        scoring = \"roc_auc\",\n",
    "                        verbose = -1,\n",
    "                        n_jobs = -1)\n",
    "    \n",
    "    # Fit GridSearchCV\n",
    "    grid.fit(X_train, y_train)\n",
    "    \n",
    "    # Making prediction on test set\n",
    "    y_pred = grid.predict_proba(X_test)[:,1]\n",
    "    \n",
    "    # Getting AUC\n",
    "    auc = roc_auc_score(y_test, y_pred)\n",
    "    \n",
    "    # Print results\n",
    "    print(f\"AUC of stack {combo}: {auc:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "a5f8168a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-11T03:05:25.470709Z",
     "iopub.status.busy": "2022-01-11T03:05:25.470023Z",
     "iopub.status.idle": "2022-01-11T03:05:25.472772Z",
     "shell.execute_reply": "2022-01-11T03:05:25.473168Z"
    },
    "papermill": {
     "duration": 0.14689,
     "end_time": "2022-01-11T03:05:25.473295",
     "exception": false,
     "start_time": "2022-01-11T03:05:25.326405",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('pipe1', 'pipe2')\n",
      "('pipe1', 'pipe3')\n",
      "('pipe2', 'pipe3')\n",
      "('pipe1', 'pipe2', 'pipe3')\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "# Classifier labels\n",
    "classifier_labels = [\"pipe1\",\"pipe2\",\"pipe3\"]\n",
    "\n",
    "# Get all unique combinations of classifier with a set size greater than or equal to 2\n",
    "combo_classifiers = []\n",
    "for ii in range(2, len(classifier_labels)+1):\n",
    "    for subset in itertools.combinations(classifier_labels, ii):\n",
    "        combo_classifiers.append(subset)\n",
    "# Stack, tune, and evaluate stack of classifiers\n",
    "for combo in combo_classifiers:\n",
    "    # Get labels of classifier to create a stack\n",
    "    print(combo)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2743c4e",
   "metadata": {
    "papermill": {
     "duration": 0.077624,
     "end_time": "2022-01-11T03:05:25.628766",
     "exception": false,
     "start_time": "2022-01-11T03:05:25.551142",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "    AUC of stack ('pipe1', 'pipe2'): 0.867\n",
    "    AUC of stack ('pipe1', 'pipe3'): 0.867\n",
    "    AUC of stack ('pipe1', 'pipe4'): 0.867\n",
    "    AUC of stack ('pipe2', 'pipe3'): 0.867\n",
    "    AUC of stack ('pipe2', 'pipe4'): 0.867\n",
    "    AUC of stack ('pipe3', 'pipe4'): 0.867\n",
    "    AUC of stack ('pipe1', 'pipe2', 'pipe3'): 0.867\n",
    "    AUC of stack ('pipe1', 'pipe2', 'pipe4'): 0.867\n",
    "    AUC of stack ('pipe1', 'pipe3', 'pipe4'): 0.867\n",
    "    AUC of stack ('pipe2', 'pipe3', 'pipe4'): 0.867"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a703b5bd",
   "metadata": {
    "papermill": {
     "duration": 0.077521,
     "end_time": "2022-01-11T03:05:25.784027",
     "exception": false,
     "start_time": "2022-01-11T03:05:25.706506",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Conclusion: \n",
    "\n",
    "Ensemble learning works best when the base models are not correlated. For instance, you can train different models such as linear models, decision trees, and neural nets on different datasets or features. The less correlated the base models, the better. \n",
    "\n",
    "The idea behind using uncorrelated models is that each may be solving a weakness of the other. They also have different strengths which, when combined, will result in a well-performing estimator. For example, creating an ensemble of just tree-based models may not be as effective as combining tree-type algorithms with other types of algorithms. \n",
    "\n",
    "## Advantages and Disadvantages of Stacking\n",
    "\n",
    "Like all other methods in machine learning, stacking has advantages and disadvantages. Here are some of the advantages of stacking:\n",
    "\n",
    "    Stacking can yield improvements in model performance.\n",
    "    Stacking reduces variance and creates a more robust model by combining the predictions of multiple models.\n",
    "\n",
    "Keep in mind that stacking also has the following disadvantages:\n",
    "\n",
    "    Stacked models can take significantly longer to train than simpler models and require more memory.\n",
    "    Generating predictions using stacked models will usually be slower and more computationally expensive. This drawback is important to consider if you are planning to deploy a stacked model into production.\n",
    "\n",
    "credit: \n",
    "\n",
    "https://towardsdatascience.com/stacking-classifiers-for-higher-predictive-performance-566f963e4840\n",
    "\n",
    "https://www.dataquest.io/blog/introduction-to-ensembles/\n",
    "\n",
    "https://scikit-learn.org/stable/modules/ensemble.html\n",
    "\n",
    "https://machinelearningmastery.com/stacking-ensemble-machine-learning-with-python/\n",
    "\n",
    "\n",
    "https://mlfromscratch.com/model-stacking-explained/#/\n",
    "\n",
    "http://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/\n",
    "\n",
    "http://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/\n",
    "\n",
    "\n",
    "https://www.kaggle.com/bannourchaker/7ensembling-stacking-averaging-scratchvssklearn/edit/run/73916270\n",
    "\n",
    "https://www.kaggle.com/bannourchaker/averagingmodels\n",
    "\n",
    "https://mlfromscratch.com/model-stacking-explained/#/\n",
    "\n",
    "https://neptune.ai/blog/ensemble-learning-guide\n",
    "\n",
    "https://www.analyticsvidhya.com/blog/2021/08/ensemble-stacking-for-machine-learning-and-deep-learning/\n",
    "\n",
    "https://www.kaggle.com/prashant111/adaboost-classifier-tutorial\n",
    "\n",
    "https://www.kaggle.com/bannourchaker/stacking/edit/run/75840835\n",
    "\n",
    "https://www.kaggle.com/bannourchaker/8-ensembling-blending-scratchvsoop\n",
    "\n",
    "Bagging:\n",
    "\n",
    "https://vitalflux.com/bagging-classifier-python-code-example/\n",
    "\n",
    "https://www.analyticsvidhya.com/blog/2018/06/comprehensive-guide-for-ensemble-models/\n",
    "\n",
    "complete guide Stacking :\n",
    "\n",
    "https://neptune.ai/blog/ensemble-learning-guide\n",
    "\n",
    "https://www.analyticsvidhya.com/blog/2021/08/ensemble-stacking-for-machine-learning-and-deep-learning/\n",
    "\n",
    "https://www.kaggle.com/prashant111/adaboost-classifier-tutorial"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 16695.743078,
   "end_time": "2022-01-11T03:05:27.199909",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-01-10T22:27:11.456831",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}