2022-50-007-ml-project - Copy.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","cell_id":"ac98df94-54a7-4e1a-8138-7f5c7571be69","deepnote_cell_height":369,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":14322146,"execution_start":1658682577993,"source_hash":"f2620bd0","trusted":true},"outputs":[],"source":["# # This Python 3 environment comes with many helpful analytics libraries installed\n","# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n","# # For example, here's several helpful packages to load\n","\n","# import numpy as np # linear algebra\n","# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","\n","# # Input data files are available in the read-only \"../input/\" directory\n","# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n","\n","# import os\n","# for dirname, _, filenames in os.walk('/kaggle/input'):\n","#     for filename in filenames:\n","#         print(os.path.join(dirname, filename))\n","\n","# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n","# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"]},{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[],"source":["import time\n","start_time = time.time()"]},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Process Initialised\n"]}],"source":["print(\"Process Initialised\")"]},{"cell_type":"markdown","metadata":{"cell_id":"00001-86291d65-f0ea-4c22-8d29-15f5f0273874","deepnote_cell_height":82,"deepnote_cell_type":"markdown"},"source":["# Load Training Dataset"]},{"cell_type":"code","execution_count":1,"metadata":{"cell_id":"eef26f675db14448bd91c5b241573c9a","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":11860273,"execution_start":1658682577995,"source_hash":"b7dcc9c8","tags":[],"trusted":true},"outputs":[],"source":["import numpy as np\n","import pandas as pd"]},{"cell_type":"code","execution_count":2,"metadata":{"cell_id":"00002-a0331af6-c3fd-496c-b8c9-2e9fbcf41caa","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":1,"execution_start":1658682578049,"source_hash":"3f76624c","trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>17180</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>17181</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>17182</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>17183</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>17184</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5002 columns</p>\n","</div>"],"text/plain":["          id  label    0    1    2    3    4    5    6    7  ...  4990  4991  \\\n","0          1      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1          2      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2          3      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3          4      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4          5      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...      ...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  17180      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  17181      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  17182      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  17183      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  17184      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5002 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 18 s\n","Wall time: 18.1 s\n"]}],"source":["%%time\n","df_train = pd.read_csv(r\"./source/train_tfidf_features.csv\")\n","display(df_train)"]},{"cell_type":"code","execution_count":6,"metadata":{"cell_id":"240e16872bb04256995ffbe41d9281ad","deepnote_cell_height":634.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":19420,"execution_start":1658682578050,"source_hash":"50b848d7","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 0 ns\n","Wall time: 0 ns\n"]}],"source":["%%time\n","# df_train = pd.read_csv(\"/work/50007-2022/train_tfidf_features.csv\")\n","# display(df_train)"]},{"cell_type":"code","execution_count":7,"metadata":{"cell_id":"00003-d747b669-d32c-43a4-b111-dea206d21537","deepnote_cell_height":534.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":16711,"execution_start":1658682597472,"source_hash":"849ce5a8","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 9.69 s\n","Wall time: 9.74 s\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>...</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>8592.500000</td>\n","      <td>0.381227</td>\n","      <td>0.000150</td>\n","      <td>0.001066</td>\n","      <td>0.001532</td>\n","      <td>0.000369</td>\n","      <td>0.000140</td>\n","      <td>0.000066</td>\n","      <td>0.000270</td>\n","      <td>0.000483</td>\n","      <td>...</td>\n","      <td>0.000202</td>\n","      <td>0.000429</td>\n","      <td>0.000286</td>\n","      <td>0.000075</td>\n","      <td>0.000260</td>\n","      <td>0.000709</td>\n","      <td>0.000257</td>\n","      <td>0.000121</td>\n","      <td>0.000308</td>\n","      <td>0.000159</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>4960.737848</td>\n","      <td>0.485702</td>\n","      <td>0.008297</td>\n","      <td>0.019532</td>\n","      <td>0.024741</td>\n","      <td>0.012334</td>\n","      <td>0.008276</td>\n","      <td>0.005065</td>\n","      <td>0.009907</td>\n","      <td>0.013106</td>\n","      <td>...</td>\n","      <td>0.010215</td>\n","      <td>0.013178</td>\n","      <td>0.011378</td>\n","      <td>0.005866</td>\n","      <td>0.010864</td>\n","      <td>0.017641</td>\n","      <td>0.010246</td>\n","      <td>0.006529</td>\n","      <td>0.010526</td>\n","      <td>0.008536</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>1.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>4296.750000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>8592.500000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>12888.250000</td>\n","      <td>1.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>17184.000000</td>\n","      <td>1.000000</td>\n","      <td>0.676327</td>\n","      <td>0.560830</td>\n","      <td>0.958430</td>\n","      <td>0.646740</td>\n","      <td>0.532789</td>\n","      <td>0.437760</td>\n","      <td>0.435835</td>\n","      <td>0.536746</td>\n","      <td>...</td>\n","      <td>0.611122</td>\n","      <td>0.540809</td>\n","      <td>0.566613</td>\n","      <td>0.592170</td>\n","      <td>0.617341</td>\n","      <td>0.850605</td>\n","      <td>0.484908</td>\n","      <td>0.398105</td>\n","      <td>0.430031</td>\n","      <td>0.528556</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 5002 columns</p>\n","</div>"],"text/plain":["                 id         label             0             1             2  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean    8592.500000      0.381227      0.000150      0.001066      0.001532   \n","std     4960.737848      0.485702      0.008297      0.019532      0.024741   \n","min        1.000000      0.000000      0.000000      0.000000      0.000000   \n","25%     4296.750000      0.000000      0.000000      0.000000      0.000000   \n","50%     8592.500000      0.000000      0.000000      0.000000      0.000000   \n","75%    12888.250000      1.000000      0.000000      0.000000      0.000000   \n","max    17184.000000      1.000000      0.676327      0.560830      0.958430   \n","\n","                  3             4             5             6             7  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000369      0.000140      0.000066      0.000270      0.000483   \n","std        0.012334      0.008276      0.005065      0.009907      0.013106   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.646740      0.532789      0.437760      0.435835      0.536746   \n","\n","       ...          4990          4991          4992          4993  \\\n","count  ...  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean   ...      0.000202      0.000429      0.000286      0.000075   \n","std    ...      0.010215      0.013178      0.011378      0.005866   \n","min    ...      0.000000      0.000000      0.000000      0.000000   \n","25%    ...      0.000000      0.000000      0.000000      0.000000   \n","50%    ...      0.000000      0.000000      0.000000      0.000000   \n","75%    ...      0.000000      0.000000      0.000000      0.000000   \n","max    ...      0.611122      0.540809      0.566613      0.592170   \n","\n","               4994          4995          4996          4997          4998  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000260      0.000709      0.000257      0.000121      0.000308   \n","std        0.010864      0.017641      0.010246      0.006529      0.010526   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.617341      0.850605      0.484908      0.398105      0.430031   \n","\n","               4999  \n","count  17184.000000  \n","mean       0.000159  \n","std        0.008536  \n","min        0.000000  \n","25%        0.000000  \n","50%        0.000000  \n","75%        0.000000  \n","max        0.528556  \n","\n","[8 rows x 5002 columns]"]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","df_train.describe()"]},{"cell_type":"markdown","metadata":{},"source":["# Task 1: Implement Logistic Regression\n","Recalled that you have learned about Logistic Regression in your earlier class. Your task is to implement a Logistic Regression model from scratch. \\\n","Note that you are NOT TO USE the sklearn logistic regression package or any other pre-defined logistic regression package for this task! \\\n","Usage of any logistic regression packages will result in 0 marks for this task.\n","\n","## Key Task Deliverables\n","1a. Code implementation of the Logistic Regression model. \\\n","1b. Prediction made by your Logistic Regression on the Test set. Note that you are welcome to submit your predicted labels to Kaggle but you will need to submit the final prediction output in the final project submission. Please label the file as \"LogRed_Prediction.csv\"."]},{"cell_type":"markdown","metadata":{"cell_id":"00005-49b9c0d6-4381-409b-82b2-2a298f466e01","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown"},"source":["-- `sigmoid(z)`: A function that takes in a Real Number input and returns an output value between 0 and 1."]},{"cell_type":"code","execution_count":8,"metadata":{"cell_id":"00006-2e3c692b-00a7-4160-acbf-0da222258054","deepnote_cell_height":135,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658687327448,"source_hash":"f9e00733","trusted":true},"outputs":[],"source":["def sigmoid(z):\n","    result = 1/(1 + np.exp(-z))\n","#     print(f\"sigmoid: {result}\")\n","    return result"]},{"cell_type":"markdown","metadata":{"cell_id":"00007-968b9856-b6f3-40bd-9301-356dee468412","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown"},"source":["-- `loss(y, y_hat)`: A loss function that allows us to minimize and determine the optimal parameters. The function takes in the actual labels y and the predicted labels yhat, and returns the overall training loss. Note that you should be using the Log Loss function taught in class."]},{"cell_type":"markdown","metadata":{"cell_id":"3bd8cc27f03146c58682dabe9bdefae8","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["Note: We have decided the add a regulariser (denoted by the `lmb` term) to observe whether there is an improvement in utilising L2 regularisation in our Logisitic Regression Model. As such, we have decided to change the arguments of the function  to accomodate for regularisation."]},{"cell_type":"code","execution_count":9,"metadata":{"cell_id":"00008-79c60bbd-81a5-460c-91a3-d3d8b7cbfd6b","deepnote_cell_height":333,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658687327449,"source_hash":"296019a7","trusted":true},"outputs":[],"source":["# def loss(y, y_hat):\n","#         result = -np.mean(y * np.log(y_hat) + (1 - y) * np.log(1 - y_hat))\n","#         print(f\"loss: {result}\")\n","#         return result\n","\n","def loss(y, X, w, b, lmb):\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    m = np.shape(y)[0]\n","    \n","    loss = -1 * np.where(y == 1, np.log(y_hat), np.log(1 - y_hat)).mean()\n","    reg = lmb * np.sum(w**2) / (2 * m)\n","    error = loss + reg\n","    \n","#     print(f\"training loss = {loss}, regularisation term = {reg}, training error = {error}\")\n","    return error"]},{"cell_type":"markdown","metadata":{"cell_id":"00009-9fef5aa0-38be-400f-a766-b0fbdaa5d8d6","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown"},"source":["-- `gradients(X, y, y_hat)`: The Gradient Descent Algorithm to find the optimal values of our parameters. The function takes in the training feature X, actual labels y and the predicted labels yhat, and returns the partial derivative of the Loss function with respect to weights (dw) and bias (db)."]},{"cell_type":"markdown","metadata":{"cell_id":"5d4d961b83aa42039d5853a59939b89a","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Likewise, the arguments of the `gradients` function has been altered to accommodate for L2 regularisation."]},{"cell_type":"code","execution_count":10,"metadata":{"cell_id":"00010-20d9c8e4-9d38-4ed5-95f4-3c2fd6c26fb1","deepnote_cell_height":243,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":2,"execution_start":1658687327454,"source_hash":"8919acd2","trusted":true},"outputs":[],"source":["def gradients(y, X, w, b, lmb):\n","    # m - number of training examples\n","    m = np.shape(X)[0]\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    \n","    dw = (1 / m) * (np.dot(X.T, (y_hat - y)) + lmb * w)\n","    db = (1 / m) * np.sum((y_hat - y))\n","    \n","#     print(f\"dw: {dw}, db: {db}\")\n","    return dw, db"]},{"cell_type":"markdown","metadata":{"cell_id":"00011-2af85165-6958-44ec-a0d5-2fa2a2a13a7c","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown"},"source":["-- `train(X, y, bs, epochs, lr)`: The training function for your model."]},{"cell_type":"markdown","metadata":{"cell_id":"59627bc57f214e5aab30f859f0436502","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["We added the `C` term to represent the penalty term `lmb`. The relationship is that `lmb = 1/C` if `C != 0`. Otherwise,`lmb = 0`, where we do not apply regularisation."]},{"cell_type":"code","execution_count":11,"metadata":{"cell_id":"00012-1ef81ca6-ff76-4273-8db1-e073ae3b5b60","deepnote_cell_height":1377,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":42,"execution_start":1658685590567,"source_hash":"44826cb7","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = np.zeros((d, 1))\n","    b = 0\n","#     w = rng.uniform(size=(d,1))\n","#     b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","#                 print(w == w_new, b == b_new);\n","#                 print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"00019-ced096de-42a1-4c68-ade7-88807850e19b","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown"},"source":["-- `predict(X, w, b)`: The prediction function where you can apply your validation and test sets."]},{"cell_type":"code","execution_count":12,"metadata":{"cell_id":"00020-e9cb6e0c-03aa-42db-943c-b6bb4a9913f0","deepnote_cell_height":135,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":6,"execution_start":1658687691735,"source_hash":"7d794aee","trusted":true},"outputs":[],"source":["def predict(X, w, b):\n","    y_pred = sigmoid(np.dot(X, w) + b)\n","    pred_labels = np.array([1 if i >= 0.5 else 0 for i in y_pred])\n","    return pred_labels"]},{"cell_type":"markdown","metadata":{"cell_id":"00021-c62a3fc4-3c62-432f-9f13-64ea72f7e3da","deepnote_cell_height":212,"deepnote_cell_type":"markdown"},"source":["## Performance Evaluation\n","\n","As per the grading rubric - \"Perfect Implementation of the Logistics Regression algorithm. Successfully trained the implemented model with the train set and achieved comparative performance compared to SKLearn Logistic Regression package\", we shall compare the performance of our model with the SKLearn LogisticRegression and SGDClassifier package.\n","\n","We shall first implement a function to evaluate the accuracy of our model. The goal is to achieve a Macro-F1 score that is within 0.05 of the Macro-F1 score of the SKLearn Package"]},{"cell_type":"code","execution_count":3,"metadata":{"cell_id":"00024-700a0602-b8f9-47d5-85c9-dfb5b28bf780","deepnote_cell_height":117,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":3,"execution_start":1658687562415,"source_hash":"72d19504","trusted":true},"outputs":[],"source":["from sklearn.linear_model import LogisticRegression, SGDClassifier\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import f1_score"]},{"cell_type":"code","execution_count":14,"metadata":{"cell_id":"00022-a17b0e65-fcbb-44d1-8ded-a7923d6d255c","deepnote_cell_height":225,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":1,"execution_start":1658687514001,"source_hash":"754349d7","trusted":true},"outputs":[],"source":["def score(y, y_hat):\n","    accuracy = np.sum(y == y_hat) / np.shape(y)[0]\n","    f1score = f1_score(y, y_hat, average='macro')\n","\n","    print(f\"Accuracy: {accuracy}\")\n","    print(f\"Macro-F1 score: {f1score}\")\n","\n","    # Return Macro-F1 score of the model\n","    return f1score"]},{"cell_type":"code","execution_count":15,"metadata":{"cell_id":"81cc9876cd5a44c9bf079ddd3cc51847","deepnote_cell_height":315,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":2,"execution_start":1658685600957,"source_hash":"c6dbbe00","tags":[],"trusted":true},"outputs":[],"source":["def perform(LogReg, SGD, **scores):\n","    models = []\n","\n","    for model, score in scores.items():\n","        result = max(abs(LogReg - score), abs(SGD - score))\n","        print(f\"Model: {model}, Macro-F1 Score: {score}, Difference: {result}\")\n","\n","        if result <= 0.05:\n","            models.append(model)\n","    \n","    quality = \"Success\" if len(models) > 0 else \"Failed\"\n","    print(f\"Model {quality}\")\n","\n","    return models"]},{"cell_type":"code","execution_count":16,"metadata":{"cell_id":"00025-7a497f62-9e30-42a1-bb11-ac7f470daeb6","deepnote_cell_height":148,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":267,"execution_start":1658682615773,"source_hash":"87db3624","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X: (17184, 5000), y: (17184,)\n","CPU times: total: 141 ms\n","Wall time: 145 ms\n"]}],"source":["%%time\n","X = df_train.iloc[:, 2:5002].to_numpy()\n","y = df_train.iloc[:,1].to_numpy()\n","print(f\"X: {X.shape}, y: {y.shape}\")"]},{"cell_type":"markdown","metadata":{"cell_id":"c46261c92089447cacdad83b77304580","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Splitting of Training Data Set to Perform Internal Validation of ML Model"]},{"cell_type":"code","execution_count":17,"metadata":{"cell_id":"00026-6f193c9e-3618-4484-8208-a051961372e4","deepnote_cell_height":168,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":1079,"execution_start":1658682616031,"source_hash":"2253195f","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 5000), y_train: (12888,)\n","X_test: (4296, 5000), y_test: (4296,)\n","CPU times: total: 2.42 s\n","Wall time: 2.41 s\n"]}],"source":["%%time\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=100)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")"]},{"cell_type":"markdown","metadata":{"cell_id":"7c27a48a55ce4ae78c6976019a616738","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Initialisation of Hyperparameters for Logistic Regression Model"]},{"cell_type":"code","execution_count":18,"metadata":{"cell_id":"00027-ee4a0808-cb79-407d-9897-e37bd6ea5fe3","deepnote_cell_height":184,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":5,"execution_start":1658682617112,"source_hash":"13936e3c","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["128 16 0.1 0\n"]}],"source":["bs = 128\n","epochs = 16\n","lr = 0.1\n","C = 0\n","print(bs, epochs, lr, C)"]},{"cell_type":"markdown","metadata":{"cell_id":"565f4ff1d2444030a1be477b5bfbce67","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Evaluation of Logisitic Regression Model"]},{"cell_type":"code","execution_count":19,"metadata":{"cell_id":"5209c131890d49f7b7e89ee7958f4402","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":187364,"execution_start":1658682617138,"source_hash":"5b601220","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.00172004]\n"," [-0.02492764]\n"," [ 0.00550406]\n"," ...\n"," [ 0.00015613]\n"," [ 0.01175477]\n"," [ 0.00490417]]\n","-0.5020697905473851\n","0.6493706267318076\n","1404\n","Accuracy: 0.6259310986964618\n","Macro-F1 score: 0.3867287807981557\n","CPU times: total: 10min 2s\n","Wall time: 1min 8s\n"]},{"data":{"text/plain":["0.3867287807981557"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model = score(y_test, predict(X_test, w, b))\n","Model"]},{"cell_type":"markdown","metadata":{"cell_id":"9fbda5801d0e4375a6c0ed673aa4581d","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Evaluation of SKLearn Logistic Regression Model (LogisticRegression and SGDClassifier)"]},{"cell_type":"code","execution_count":20,"metadata":{"cell_id":"1c2de54ac3904efeaeace8e6bf90af98","deepnote_cell_height":316.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":5404,"execution_start":1658682804499,"source_hash":"3125af87","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.58187735 -1.02280517 -0.39681143 ... -0.04059766  0.49543838\n","   0.40047111]]\n","[-0.84683029]\n","Accuracy: 0.7302141527001862\n","Macro-F1 score: 0.6906115962801043\n","CPU times: total: 18.7 s\n","Wall time: 1.99 s\n"]},{"data":{"text/plain":["0.6906115962801043"]},"execution_count":20,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","clf1 = LogisticRegression(random_state = 100).fit(X_train, y_train)\n","print(clf1.coef_)\n","print(clf1.intercept_)\n","SKLearnLogReg = score(y_test, clf1.predict(X_test))\n","SKLearnLogReg"]},{"cell_type":"code","execution_count":21,"metadata":{"cell_id":"c4a0d020717347f8be43aaf0b5405db5","deepnote_cell_height":334.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":3477,"execution_start":1658682809910,"source_hash":"f7a528bd","tags":[],"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["c:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:173: FutureWarning: The loss 'log' was deprecated in v1.1 and will be removed in version 1.3. Use `loss='log_loss'` which is equivalent.\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["[[-0.47493728 -0.87421955 -0.31040593 ... -0.0494175   0.42273107\n","   0.31604817]]\n","[-0.75282034]\n","Accuracy: 0.728584729981378\n","Macro-F1 score: 0.6895107183766978\n","CPU times: total: 2.19 s\n","Wall time: 2.19 s\n"]},{"data":{"text/plain":["0.6895107183766978"]},"execution_count":21,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","clf2 = SGDClassifier(loss=\"log\", random_state=100).fit(X_train, y_train)\n","# clf2 = SGDClassifier(loss=\"log_loss\", random_state=100).fit(X_train, y_train)\n","print(clf2.coef_)\n","print(clf2.intercept_)\n","SKLearnSGD = score(y_test, clf2.predict(X_test))\n","SKLearnSGD"]},{"cell_type":"code","execution_count":22,"metadata":{"cell_id":"a0a7f0b441634a08bb9ffa3969fbe745","deepnote_cell_height":184.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":16,"execution_start":1658682813394,"source_hash":"5dbd2ce0","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: Model, Macro-F1 Score: 0.3867287807981557, Difference: 0.30388281548194856\n","Model Failed\n"]},{"data":{"text/plain":["[]"]},"execution_count":22,"metadata":{},"output_type":"execute_result"}],"source":["perform(SKLearnLogReg, SKLearnSGD, Model=Model)"]},{"cell_type":"markdown","metadata":{"cell_id":"38e4ab11c51d42afbcb9b3ce1379977a","deepnote_cell_height":214.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above results, we can observe that our Logistic Regression Model is severely underperforming as compared to the SKLearn Packages. We believe that this can be due to 2 possible reasons - The initialisation of the parameters `w` and `b` and the greedy approach adopted in navigating the gradient descent algorithm. \n","\n","We shall try a random initialisation and try 2 additional variations in developing our gradient descent algorithm (One where we choose the parameters that ensures minimum training error after every epoch and another where we choose the the parameters that ensure minimum training error at the end of the algorithm).\n","\n","We will evaluate these 3 models and compare their scores with those of the SKLearn Packages."]},{"cell_type":"markdown","metadata":{"cell_id":"1a044d9873f446408ad2aa101bb7d9af","deepnote_cell_height":70,"deepnote_cell_type":"markdown","tags":[]},"source":["## Tuning the Logistic Regression Model"]},{"cell_type":"markdown","metadata":{"cell_id":"a38debc715c64e2fa2dcb9a4f93a9479","deepnote_cell_height":62,"deepnote_cell_type":"markdown","tags":[]},"source":["### Tuning the Gradient Descent Algorithm"]},{"cell_type":"markdown","metadata":{"cell_id":"9349f4342add4e35a19655934163f24a","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train1` function, we will implement a random initialisation of parameters `w` and `b` whose values are very close to 0 using the uniform distribution [0, 1)."]},{"cell_type":"code","execution_count":23,"metadata":{"cell_id":"00014-6addd038-1b3b-427b-8ffe-cbfefb160449","deepnote_cell_height":1377,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":11,"execution_start":1658682813420,"source_hash":"7a9d88a4","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train1(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","#                 print(w == w_new, b == b_new);\n","#                 print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"6071a755307649bdb4eecc970033e4b6","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train2` function, we will implement a random initialisation of parameters `w` and `b` whose values are very close to 0 using the uniform distribution [0, 1). Also, we will only choose the parameters that ensure minimum training error only at the end of the algorithm."]},{"cell_type":"code","execution_count":24,"metadata":{"cell_id":"00013-87646db4-96ee-4ef0-8640-5664dccf1c4f","deepnote_cell_height":1305,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":7,"execution_start":1658682813446,"source_hash":"af40613f","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train2(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","\n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","\n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"0eb861086da24e4798148ff9390189c8","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train3` function, we will implement a random initialisation of parameters `w` and `b` whose values are very close to 0 using the uniform distribution [0, 1). Also, we will only choose the parameters that ensure minimum training error after every epoch."]},{"cell_type":"code","execution_count":25,"metadata":{"cell_id":"00015-9fce8db2-a072-46fa-8931-9c1bcf1e8ae3","deepnote_cell_height":1449,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":8,"execution_start":1658682813469,"source_hash":"f961edc1","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train3(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":26,"metadata":{"cell_id":"fa96d2cb0d7544ad8f6577e60c12b238","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":188007,"execution_start":1658682813480,"source_hash":"142cfe6d","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[0.82953088]\n"," [0.56112497]\n"," [0.28441661]\n"," ...\n"," [0.54737681]\n"," [0.20428492]\n"," [0.14764031]]\n","-1.7553462188993563\n","0.6582263563878277\n","1477\n","Accuracy: 0.6240689013035382\n","Macro-F1 score: 0.48659922937049865\n","CPU times: total: 10min 47s\n","Wall time: 1min 6s\n"]},{"data":{"text/plain":["0.48659922937049865"]},"execution_count":26,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train1(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model1 = score(y_test, predict(X_test, w, b))\n","Model1"]},{"cell_type":"code","execution_count":27,"metadata":{"cell_id":"a22fce9348614d9892f976e6e9fdf754","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":196553,"execution_start":1658683000816,"source_hash":"8629bebf","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[0.82917508]\n"," [0.55758287]\n"," [0.28212895]\n"," ...\n"," [0.54710171]\n"," [0.2049865 ]\n"," [0.14875909]]\n","-1.7440156421330273\n","0.6568636717097833\n","1600\n","Accuracy: 0.6243016759776536\n","Macro-F1 score: 0.4920613897314961\n","CPU times: total: 10min 1s\n","Wall time: 1min 14s\n"]},{"data":{"text/plain":["0.4920613897314961"]},"execution_count":27,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train2(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model2 = score(y_test, predict(X_test, w, b))\n","Model2"]},{"cell_type":"code","execution_count":28,"metadata":{"cell_id":"59dce1de6aff487aa46898e2e7156489","deepnote_cell_height":492,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":31925,"execution_start":1658683197315,"source_hash":"d6b6ea82","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[0.82917524]\n"," [0.55793472]\n"," [0.28220347]\n"," ...\n"," [0.5470061 ]\n"," [0.20511076]\n"," [0.14884337]]\n","-1.7438202768654791\n","0.656952130767089\n","1600\n","Accuracy: 0.6243016759776536\n","Macro-F1 score: 0.4920613897314961\n","CPU times: total: 11min 3s\n","Wall time: 1min 9s\n"]},{"data":{"text/plain":["0.4920613897314961"]},"execution_count":28,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train3(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model3 = score(y_test, predict(X_test, w, b))\n","Model3"]},{"cell_type":"code","execution_count":29,"metadata":{"cell_id":"6aa2a0555ce94cafb6a7fedc78c72891","deepnote_cell_height":262.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":2,"execution_start":1658671969346,"source_hash":"c1d0bc52","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: Model, Macro-F1 Score: 0.3867287807981557, Difference: 0.30388281548194856\n","Model: Model1, Macro-F1 Score: 0.48659922937049865, Difference: 0.2040123669096056\n","Model: Model2, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model: Model3, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model Failed\n"]},{"data":{"text/plain":["[]"]},"execution_count":29,"metadata":{},"output_type":"execute_result"}],"source":["perform(SKLearnLogReg, SKLearnSGD, Model=Model, \\\n","Model1=Model1, Model2=Model2, Model3=Model3)"]},{"cell_type":"markdown","metadata":{"cell_id":"68ffe1f2501642fdba5f5ca7c6f9c718","deepnote_cell_height":489.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above results, we can see that the random initialisation of `w` and `b` has improved the performance of the gradient descent algorithm. However, we have noted that varying the design of the gradient descent algorithm at this stage has minimal effect on the model performance.\n","\n","Upon inspection of the training dataset, we have noted that the training data is extremely sparse. Hence, we need to incorporate the idea of momentum and adaptive learning rate into our gradient descent algorithm. The introduction of momentum is to ensure that the gradient descent algorithm moves in the direction of the trend (weighted average of gradients) even in the presence of anomalous gradient values or zero gradient values. This would help accelerate the training process.\n","\n","Additionally, as a result of sparse data, we require an adaptive learning rate that is able to boost the respective parameters of `w_i` and `b` appropriately such that the algorithm is more sensitive to valuable data that are present within the training set. This would help improve the correction and accelerate the training process.\n","\n","Based on the 2 requirements, we have decided to use the `AdamW` optimiser, which uses a combination of momentum and RMSProp. Moreover, `AdamW` is an improvement over the `Adam` optimiser, in that it supports regularisation by introducing a Weight Decay component to the gradient descent algorithm. This is necessary due to the use of adaptive learning rate that skews the regularisation of parameters in the `Adam` optimiser.\n","\n","Likewise, we will introduce 3 different variants of the `AdamW` optimiser. \\\n","    1. One where we choose the parameters that ensure a smaller training error after every update (Greedy Approach) \\\n","    2. One where we choose the parameters that ensure minimum training error at the end of the algorithm \\\n","    3. One where we choose the parameters that ensures minimum training error after every epoch"]},{"cell_type":"markdown","metadata":{"cell_id":"80fa9c837cfb4b0d8459d540dd17e727","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train4` function, it is an improvement of the `train1` function. We will use the `AdamW` optimiser instead of the conventional gradient descent algorithm. We will maintain the random initialisation in this algorithm."]},{"cell_type":"code","execution_count":30,"metadata":{"cell_id":"00017-1ee80d48-ede9-4bb5-8887-503569bd3183","deepnote_cell_height":1719,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":39,"execution_start":1658671969347,"source_hash":"cd1189da","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train4(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","#                 print(w == w_new, b == b_new);\n","#                 print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"120b7bb1dfb344c58acb38d783fc6401","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train5` function, it is an improvement of the `train2` function. We will use the `AdamW` optimiser instead of the conventional gradient descent algorithm. We will maintain the random initialisation in this algorithm."]},{"cell_type":"code","execution_count":31,"metadata":{"cell_id":"00016-e5039b6e-272f-46e1-bc5a-11cafd259faf","deepnote_cell_height":1665,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":0,"execution_start":1658671969386,"source_hash":"48058778","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train5(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","\n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"eae2c6e325b646bc88b45a58a813eb3f","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train6` function, it is an improvement of the `train3` function. We will use the `AdamW` optimiser instead of the conventional gradient descent algorithm. We will maintain the random initialisation in this algorithm."]},{"cell_type":"code","execution_count":32,"metadata":{"cell_id":"00018-45469d69-b680-4896-aa5a-a35ec13eeae1","deepnote_cell_height":1791,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658687341941,"source_hash":"cee706ad","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train6(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"1a7db9c26cea4c4eb02e5d4db1fb3ece","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on our research, the recommended values of the hyperparameters for the AdamW optimizer are `beta_m = 0.9`, `beta_v = 0.999` and `err = 1e-8`."]},{"cell_type":"code","execution_count":33,"metadata":{"cell_id":"c2ea795b5d2f47ff96be49f0fd2ca797","deepnote_cell_height":238,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":4,"execution_start":1658671969407,"source_hash":"a6484cd3","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["128 16 0.1 0 0.9 0.999 1e-08\n"]}],"source":["bs = 128\n","epochs = 16\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","print(bs, epochs, lr, C, beta_m, beta_v, err)"]},{"cell_type":"code","execution_count":34,"metadata":{"cell_id":"00032-4e0a19ee-70b6-47ee-bc12-542dbcd9c1d1","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":202294,"execution_start":1658671969431,"source_hash":"aafebce6","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-8.47741643]\n"," [-8.10057621]\n"," [-4.33031263]\n"," ...\n"," [ 1.54335685]\n"," [ 3.65972176]\n"," [ 3.82466584]]\n","-1.1785629010376373\n","0.2810261570260205\n","1462\n","Accuracy: 0.6722532588454376\n","Macro-F1 score: 0.6494362017804154\n","CPU times: total: 11min 8s\n","Wall time: 1min 12s\n"]},{"data":{"text/plain":["0.6494362017804154"]},"execution_count":34,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train4(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model4 = score(y_test, predict(X_test, w, b))\n","Model4"]},{"cell_type":"code","execution_count":35,"metadata":{"cell_id":"00033-823f1f81-4f98-42b9-b244-7aa6caad1dba","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":211254,"execution_start":1658672171550,"source_hash":"d445cb11","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-9.15852804]\n"," [-8.63754918]\n"," [-4.72636096]\n"," ...\n"," [ 1.62198725]\n"," [ 3.88039144]\n"," [ 3.71492837]]\n","-1.2682779368539099\n","0.28378523874847067\n","1598\n","Accuracy: 0.6715549348230913\n","Macro-F1 score: 0.6496150778344336\n","CPU times: total: 11min 14s\n","Wall time: 1min 9s\n"]},{"data":{"text/plain":["0.6496150778344336"]},"execution_count":35,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train5(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model5 = score(y_test, predict(X_test, w, b))\n","Model5"]},{"cell_type":"code","execution_count":36,"metadata":{"cell_id":"00034-67b8b9dc-2b70-457d-8257-6d3ac7a14ac0","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":203230,"execution_start":1658672382798,"source_hash":"253fb0c6","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-8.22759819]\n"," [-8.33745542]\n"," [-3.9632901 ]\n"," ...\n"," [ 1.95402751]\n"," [ 3.52822577]\n"," [ 3.67153536]]\n","-1.1590263536783283\n","0.2840447771365454\n","1532\n","Accuracy: 0.672951582867784\n","Macro-F1 score: 0.6507565038781207\n","CPU times: total: 11min\n","Wall time: 1min 9s\n"]},{"data":{"text/plain":["0.6507565038781207"]},"execution_count":36,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model6 = score(y_test, predict(X_test, w, b))\n","Model6"]},{"cell_type":"code","execution_count":37,"metadata":{"cell_id":"bed05c58066f43559152cfe6d491fda7","deepnote_cell_height":340.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":285,"execution_start":1658672585746,"source_hash":"47f11e70","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: Model, Macro-F1 Score: 0.3867287807981557, Difference: 0.30388281548194856\n","Model: Model1, Macro-F1 Score: 0.48659922937049865, Difference: 0.2040123669096056\n","Model: Model2, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model: Model3, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model: Model4, Macro-F1 Score: 0.6494362017804154, Difference: 0.04117539449968888\n","Model: Model5, Macro-F1 Score: 0.6496150778344336, Difference: 0.04099651844567065\n","Model: Model6, Macro-F1 Score: 0.6507565038781207, Difference: 0.039855092401983594\n","Model Success\n"]},{"data":{"text/plain":["['Model4', 'Model5', 'Model6']"]},"execution_count":37,"metadata":{},"output_type":"execute_result"}],"source":["perform(SKLearnLogReg, SKLearnSGD, Model=Model, \\\n","Model1=Model1, Model2=Model2, Model3=Model3, \\\n","Model4=Model4, Model5=Model5, Model6=Model6)"]},{"cell_type":"markdown","metadata":{"cell_id":"00052-0bd975ab-19ef-4aff-86d1-65a85656d013","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown"},"source":["Based on the above score, we can deem that the performance of our Logistic Regression Model is comparable to that of SKLearn Logistic Regression Package. Moreover, our best performing model is Model6. Hence, we shall refine our Logistic Regression Model by tuning the hyperparameters."]},{"cell_type":"markdown","metadata":{"cell_id":"ae11d50cb29e4b888746694fde2f6fe7","deepnote_cell_height":62,"deepnote_cell_type":"markdown","tags":[]},"source":["### Hyperparameter Tuning"]},{"cell_type":"markdown","metadata":{"cell_id":"1b5f0d3cd5a84c7a8139b176069da8fe","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["To further improve our model performance, we will first find the best learning rate, regularisation coefficient, momentum coefficient, RMSProp coefficient and the Error Term. Once we have chosen the optimal hyperparameters, we will then decide on the Batch Size and Epoch Size to train our model."]},{"cell_type":"code","execution_count":38,"metadata":{"cell_id":"00035-fc897196-e51a-4ee4-9bd3-f9063c3d47c6","deepnote_cell_height":372,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":233,"execution_start":1658677091895,"source_hash":"8b69acce","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Learning Rates: [1, 0.1, 0.01, 0.001]\n","Regularisation Coefficients: [0.1, 1, 10, 100, 1000, 0]\n","Momentum Coefficients: [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.0]\n","RMSProp Coefficients: [0.999, 0.997, 0.995, 0.993, 0.991, 0.989]\n","Error Terms: [1, 0.01, 0.0001, 1e-06, 1e-08]\n"]}],"source":["Ls = [10**(-i) for i in range(4)]\n","Cs = [0.1, 1, 10, 100, 1000, 0]\n","Bm = [(9 - i)/10 for i in range(10)]\n","Bv = [(999 - 2*i)/1000 for i in range(6)]\n","Errs = [10**(-2*i) for i in range(5)]\n","\n","print(f\"Learning Rates: {Ls}\")\n","print(f\"Regularisation Coefficients: {Cs}\")\n","print(f\"Momentum Coefficients: {Bm}\")\n","print(f\"RMSProp Coefficients: {Bv}\")\n","print(f\"Error Terms: {Errs}\")"]},{"cell_type":"code","execution_count":39,"metadata":{"cell_id":"00037-9aee6fca-0854-4294-bc7f-d7ec6603904e","deepnote_cell_height":848,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":189719,"execution_start":1658675006802,"source_hash":"7c0af1ce","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6538640595903166\n","Macro-F1 score: 0.60183066275978\n","303\n","1 0 0.9 0.999 1e-08 0.60183066275978\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","Accuracy: 0.6778398510242085\n","Macro-F1 score: 0.6537110228158275\n","400\n","0.01 0 0.9 0.999 1e-08 0.6537110228158275\n","Accuracy: 0.3743016759776536\n","Macro-F1 score: 0.2723577235772358\n","400\n","0.001 0 0.9 0.999 1e-08 0.2723577235772358\n","{0.60183066275978: 1, 0.6662472846585037: 0.1, 0.6537110228158275: 0.01, 0.2723577235772358: 0.001}\n","[0.1] 0.6662472846585037\n","CPU times: total: 10min 54s\n","Wall time: 1min 11s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for lr in Ls:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = lr\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"2bd880bb5f8d45159419ada1dbf55c62","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a learning rate of `lr = 0.1`."]},{"cell_type":"code","execution_count":40,"metadata":{"cell_id":"49aed85a49d6433bb21c9f510a9c61ef","deepnote_cell_height":1012.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":301417,"execution_start":1658673145534,"source_hash":"48c55687","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","320\n","0.1 0.1 0.9 0.999 1e-08 0.384791636832307\n","Accuracy: 0.6261638733705773\n","Macro-F1 score: 0.3879851144518121\n","323\n","0.1 1 0.9 0.999 1e-08 0.3879851144518121\n","Accuracy: 0.680633147113594\n","Macro-F1 score: 0.5902111978259219\n","338\n","0.1 10 0.9 0.999 1e-08 0.5902111978259219\n","Accuracy: 0.7178770949720671\n","Macro-F1 score: 0.6847822296614012\n","315\n","0.1 100 0.9 0.999 1e-08 0.6847822296614012\n","Accuracy: 0.699487895716946\n","Macro-F1 score: 0.6741020618444631\n","400\n","0.1 1000 0.9 0.999 1e-08 0.6741020618444631\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","{0.384791636832307: 0.1, 0.3879851144518121: 1, 0.5902111978259219: 10, 0.6847822296614012: 100, 0.6741020618444631: 1000, 0.6662472846585037: 0}\n","[100] 0.6847822296614012\n","CPU times: total: 15min 46s\n","Wall time: 1min 40s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for C in Cs:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = C\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"654fc227f0a3475496364d46011a61fd","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a regularisation coefficient of `C = 100`."]},{"cell_type":"code","execution_count":41,"metadata":{"cell_id":"ef8f9fd43d2347328baa1bc48f2c59d9","deepnote_cell_height":1097,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":532213,"execution_start":1658678203178,"source_hash":"994d17cb","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","Accuracy: 0.6904096834264432\n","Macro-F1 score: 0.6673977263492802\n","400\n","0.1 0 0.8 0.999 1e-08 0.6673977263492802\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6668398896756464\n","400\n","0.1 0 0.7 0.999 1e-08 0.6668398896756464\n","Accuracy: 0.6908752327746741\n","Macro-F1 score: 0.6674584588869584\n","400\n","0.1 0 0.6 0.999 1e-08 0.6674584588869584\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6664851322495824\n","400\n","0.1 0 0.5 0.999 1e-08 0.6664851322495824\n","Accuracy: 0.6911080074487895\n","Macro-F1 score: 0.6672200359410805\n","400\n","0.1 0 0.4 0.999 1e-08 0.6672200359410805\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6673659689525142\n","398\n","0.1 0 0.3 0.999 1e-08 0.6673659689525142\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6673659689525142\n","398\n","0.1 0 0.2 0.999 1e-08 0.6673659689525142\n","Accuracy: 0.6894785847299814\n","Macro-F1 score: 0.6666598418821515\n","398\n","0.1 0 0.1 0.999 1e-08 0.6666598418821515\n","Accuracy: 0.688780260707635\n","Macro-F1 score: 0.6660410286078096\n","398\n","0.1 0 0.0 0.999 1e-08 0.6660410286078096\n","{0.6662472846585037: 0.9, 0.6673977263492802: 0.8, 0.6668398896756464: 0.7, 0.6674584588869584: 0.6, 0.6664851322495824: 0.5, 0.6672200359410805: 0.4, 0.6673659689525142: 0.2, 0.6666598418821515: 0.1, 0.6660410286078096: 0.0}\n","[0.6] 0.6674584588869584\n","CPU times: total: 27min 6s\n","Wall time: 2min 43s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for beta_m in Bm:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = beta_m\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"c7c1bbe3b3134a8a9ec62f6210a7a5d1","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a Momentum coefficient of `beta_m = 0.6`."]},{"cell_type":"code","execution_count":42,"metadata":{"cell_id":"da16fd14ed2246e18f62329dfc8047b6","deepnote_cell_height":1012.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":305369,"execution_start":1658678735406,"source_hash":"2f130d35","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","Accuracy: 0.6885474860335196\n","Macro-F1 score: 0.6656603211681549\n","400\n","0.1 0 0.9 0.997 1e-08 0.6656603211681549\n","Accuracy: 0.6894785847299814\n","Macro-F1 score: 0.6661331817080218\n","400\n","0.1 0 0.9 0.995 1e-08 0.6661331817080218\n","Accuracy: 0.6885474860335196\n","Macro-F1 score: 0.6651320818030984\n","400\n","0.1 0 0.9 0.993 1e-08 0.6651320818030984\n","Accuracy: 0.6883147113594041\n","Macro-F1 score: 0.6646593875910989\n","400\n","0.1 0 0.9 0.991 1e-08 0.6646593875910989\n","Accuracy: 0.688780260707635\n","Macro-F1 score: 0.6650710789626108\n","400\n","0.1 0 0.9 0.989 1e-08 0.6650710789626108\n","{0.6662472846585037: 0.999, 0.6656603211681549: 0.997, 0.6661331817080218: 0.995, 0.6651320818030984: 0.993, 0.6646593875910989: 0.991, 0.6650710789626108: 0.989}\n","[0.999] 0.6662472846585037\n","CPU times: total: 16min 23s\n","Wall time: 1min 37s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for beta_v in Bv:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = beta_v\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"8d44fa87a5b741daaf184e4b1c9cefa3","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a RMSProp coefficient of `beta_v = 0.999`."]},{"cell_type":"code","execution_count":43,"metadata":{"cell_id":"9a061a8962f545b487bd86565bd3326b","deepnote_cell_height":928,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":240096,"execution_start":1658673935844,"source_hash":"61d62e58","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.41783054003724396\n","Macro-F1 score: 0.3849706750029923\n","400\n","0.1 0 0.9 0.999 1 0.3849706750029923\n","Accuracy: 0.7178770949720671\n","Macro-F1 score: 0.677383393357214\n","400\n","0.1 0 0.9 0.999 0.01 0.677383393357214\n","Accuracy: 0.6978584729981379\n","Macro-F1 score: 0.6746235436795763\n","400\n","0.1 0 0.9 0.999 0.0001 0.6746235436795763\n","Accuracy: 0.6892458100558659\n","Macro-F1 score: 0.6664535557239946\n","400\n","0.1 0 0.9 0.999 1e-06 0.6664535557239946\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","{0.3849706750029923: 1, 0.677383393357214: 0.01, 0.6746235436795763: 0.0001, 0.6664535557239946: 1e-06, 0.6662472846585037: 1e-08}\n","[0.01] 0.677383393357214\n","CPU times: total: 13min 43s\n","Wall time: 1min 20s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for err in Errs:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = err\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"399c9be926c8452a849f1915e51d7eda","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose an Error Term of `err = 0.01`."]},{"cell_type":"markdown","metadata":{"cell_id":"e180cf97009646918155b28b3ed4f16e","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Assuming that the hyperparameters are independent of one another, we will now decide on the Batch Size and Epoch Size to train our model."]},{"cell_type":"code","execution_count":44,"metadata":{"cell_id":"1d54624481ed4d52a55a467afccd3320","deepnote_cell_height":222,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":2,"execution_start":1658681589053,"source_hash":"f873e1e6","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Batch Sizes: [4096, 2048, 1024, 512, 256, 128, 64]\n","Epoch Sizes: [1, 2, 4, 8, 16, 32, 64, 128]\n"]}],"source":["Bs = [64*(2**i) for i in range(7)]\n","Bs.reverse()\n","Es = [2**(i) for i in range(8)]\n","\n","print(f\"Batch Sizes: {Bs}\")\n","print(f\"Epoch Sizes: {Es}\")"]},{"cell_type":"code","execution_count":45,"metadata":{"cell_id":"6299f80d74ff43aa91fac42121dc9712","deepnote_cell_height":934.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":372691,"execution_start":1658677441479,"source_hash":"6153f8ec","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.3961824953445065\n","Macro-F1 score: 0.32737814572174595\n","4096 8 0.1 100 0.6 0.999 0.01 0.32737814572174595\n","Accuracy: 0.4590316573556797\n","Macro-F1 score: 0.45136005106620936\n","2048 8 0.1 100 0.6 0.999 0.01 0.45136005106620936\n","Accuracy: 0.5833333333333334\n","Macro-F1 score: 0.5710142883665134\n","1024 8 0.1 100 0.6 0.999 0.01 0.5710142883665134\n","Accuracy: 0.6820297951582868\n","Macro-F1 score: 0.6424082898028336\n","512 8 0.1 100 0.6 0.999 0.01 0.6424082898028336\n","Accuracy: 0.7176443202979516\n","Macro-F1 score: 0.6713070335546949\n","256 8 0.1 100 0.6 0.999 0.01 0.6713070335546949\n","Accuracy: 0.7225325884543762\n","Macro-F1 score: 0.6757364652101494\n","128 8 0.1 100 0.6 0.999 0.01 0.6757364652101494\n","Accuracy: 0.7155493482309124\n","Macro-F1 score: 0.6692042924893535\n","64 8 0.1 100 0.6 0.999 0.01 0.6692042924893535\n","{0.32737814572174595: (4096, 8), 0.45136005106620936: (2048, 8), 0.5710142883665134: (1024, 8), 0.6424082898028336: (512, 8), 0.6713070335546949: (256, 8), 0.6757364652101494: (128, 8), 0.6692042924893535: (64, 8)}\n","[(128, 8)] 0.6757364652101494\n","CPU times: total: 21min 45s\n","Wall time: 2min 13s\n"]}],"source":["%%time\n","vals = {}\n","epochs = 8\n","lr = 0.1\n","C = 100\n","beta_m = 0.6\n","beta_v = 0.999\n","err = 0.01\n","\n","for bs in Bs:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    # print(l.index(min(l)))\n","    print(bs, epochs, lr, C, beta_m, beta_v, err, result)\n","    vals[result] = (bs, epochs)\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"d1d28c1877104b2688bed296c91551dc","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above, we can observe that for fixed epoch, the general trend is that the smaller the batch size, the better the performance. However, we have noted that the performance drops when the batch size decreases from 128 to 64, implying that the model may have overfitted with the training dataset."]},{"cell_type":"code","execution_count":46,"metadata":{"cell_id":"00036-3b9435e2-6c4c-4e47-b4db-98d47bc454b7","deepnote_cell_height":934.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":852126,"execution_start":1658679040793,"source_hash":"74277725","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.728584729981378\n","Macro-F1 score: 0.6950824941075637\n","4096 256 0.1 100 0.6 0.999 0.01 0.6950824941075637\n","Accuracy: 0.7316108007448789\n","Macro-F1 score: 0.6966186243774475\n","2048 128 0.1 100 0.6 0.999 0.01 0.6966186243774475\n","Accuracy: 0.729050279329609\n","Macro-F1 score: 0.6937789760740024\n","1024 64 0.1 100 0.6 0.999 0.01 0.6937789760740024\n","Accuracy: 0.7309124767225326\n","Macro-F1 score: 0.6957748632444369\n","512 32 0.1 100 0.6 0.999 0.01 0.6957748632444369\n","Accuracy: 0.728584729981378\n","Macro-F1 score: 0.6903260458584444\n","256 16 0.1 100 0.6 0.999 0.01 0.6903260458584444\n","Accuracy: 0.7225325884543762\n","Macro-F1 score: 0.6757364652101494\n","128 8 0.1 100 0.6 0.999 0.01 0.6757364652101494\n","Accuracy: 0.7167132216014898\n","Macro-F1 score: 0.6630075908232557\n","64 4 0.1 100 0.6 0.999 0.01 0.6630075908232557\n","{0.6950824941075637: (4096, 256), 0.6966186243774475: (2048, 128), 0.6937789760740024: (1024, 64), 0.6957748632444369: (512, 32), 0.6903260458584444: (256, 16), 0.6757364652101494: (128, 8), 0.6630075908232557: (64, 4)}\n","[(2048, 128)] 0.6966186243774475\n","CPU times: total: 41min 58s\n","Wall time: 5min 7s\n"]}],"source":["%%time\n","vals = {}\n","lr = 0.1\n","C = 100\n","beta_m = 0.6\n","beta_v = 0.999\n","err = 0.01\n","\n","for bs in Bs:\n","    epochs = bs // 16\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    # print(l.index(min(l)))\n","    print(bs, epochs, lr, C, beta_m, beta_v, err, result)\n","    vals[result] = (bs, epochs)\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"aa8adfffda094951a754372820042fa1","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above, a batch size of `bs = 2048` will result in the best model performance."]},{"cell_type":"code","execution_count":47,"metadata":{"cell_id":"949382c344aa4e0b81afbfc266a29986","deepnote_cell_height":994.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":255644,"execution_start":1658681600341,"source_hash":"b875ab39","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.3745344506517691\n","Macro-F1 score: 0.2724809483488569\n","2048 1 0.1 100 0.6 0.999 0.01 0.2724809483488569\n","Accuracy: 0.37756052141527\n","Macro-F1 score: 0.28019622016520107\n","2048 2 0.1 100 0.6 0.999 0.01 0.28019622016520107\n","Accuracy: 0.3957169459962756\n","Macro-F1 score: 0.32663601058793984\n","2048 4 0.1 100 0.6 0.999 0.01 0.32663601058793984\n","Accuracy: 0.4590316573556797\n","Macro-F1 score: 0.45136005106620936\n","2048 8 0.1 100 0.6 0.999 0.01 0.45136005106620936\n","Accuracy: 0.5833333333333334\n","Macro-F1 score: 0.5703066677276047\n","2048 16 0.1 100 0.6 0.999 0.01 0.5703066677276047\n","Accuracy: 0.675512104283054\n","Macro-F1 score: 0.6373252069053643\n","2048 32 0.1 100 0.6 0.999 0.01 0.6373252069053643\n","Accuracy: 0.7206703910614525\n","Macro-F1 score: 0.6803361332031991\n","2048 64 0.1 100 0.6 0.999 0.01 0.6803361332031991\n","Accuracy: 0.7316108007448789\n","Macro-F1 score: 0.6966186243774475\n","2048 128 0.1 100 0.6 0.999 0.01 0.6966186243774475\n","{0.2724809483488569: (2048, 1), 0.28019622016520107: (2048, 2), 0.32663601058793984: (2048, 4), 0.45136005106620936: (2048, 8), 0.5703066677276047: (2048, 16), 0.6373252069053643: (2048, 32), 0.6803361332031991: (2048, 64), 0.6966186243774475: (2048, 128)}\n","[(2048, 128)] 0.6966186243774475\n","CPU times: total: 12min 50s\n","Wall time: 1min 46s\n"]}],"source":["%%time\n","vals = {}\n","bs = 2048\n","lr = 0.1\n","C = 100\n","beta_m = 0.6\n","beta_v = 0.999\n","err = 0.01\n","\n","for epochs in Es:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    # print(l.index(min(l)))\n","    print(bs, epochs, lr, C, beta_m, beta_v, err, result)\n","    vals[result] = (bs, epochs)\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"483816e99eca4b4296d6a329b5f2d5dd","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above, we can clearly see that a larger epoch size will improve the model performance. However, there is a tradeoff in terms of the time spent to train the model. As such, we shall choose an epoch size of `epochs = 256` to train our model."]},{"cell_type":"code","execution_count":48,"metadata":{"cell_id":"77b57ac8664f47079777bdb4f59f0959","deepnote_cell_height":189,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":2,"execution_start":1658683807833,"source_hash":"1d06537f","tags":[],"trusted":true},"outputs":[],"source":["bs = 2048\n","epochs = 256\n","lr = 0.1 #best: 0.1 (btw 0.10 and 0.16)\n","C = 100 #best: 100\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)"]},{"cell_type":"code","execution_count":49,"metadata":{"cell_id":"1030069b017a421c99dfc46ae7cbfde7","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":275476,"execution_start":1658683811716,"source_hash":"7abb0f5","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.15609145]\n"," [-1.1398634 ]\n"," [-0.47299625]\n"," ...\n"," [ 0.31361622]\n"," [ 0.59016236]\n"," [ 0.52905051]]\n","-1.3197810454686933\n","0.43972440359598974\n","1536\n","Accuracy: 0.7358007448789572\n","Macro-F1 score: 0.7062293681825118\n","CPU times: total: 12min 48s\n","Wall time: 1min 46s\n"]},{"data":{"text/plain":["0.7062293681825118"]},"execution_count":49,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model7 = score(y_test, predict(X_test, w, b))\n","Model7"]},{"cell_type":"code","execution_count":39,"metadata":{"trusted":true},"outputs":[],"source":["from sklearn.preprocessing import RobustScaler as Scaler"]},{"cell_type":"code","execution_count":51,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X: (17184, 5000), y: (17184,)\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5000 columns</p>\n","</div>"],"text/plain":["       0     1     2     3     4     5     6     7     8     9     ...  4990  \\\n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","...     ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  ...   ...   \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n","\n","       4991  4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5000 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["X = df_train.iloc[:, 2:5002].to_numpy()\n","y = df_train.iloc[:,1].to_numpy()\n","print(f\"X: {X.shape}, y: {y.shape}\")\n","\n","scaler = Scaler(with_centering=False).fit(X)\n","X_scaler = scaler.transform(X)\n","display(pd.DataFrame(X_scaler))"]},{"cell_type":"code","execution_count":52,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 5000), y_train: (12888,)\n","X_test: (4296, 5000), y_test: (4296,)\n","CPU times: total: 2.41 s\n","Wall time: 2.42 s\n"]}],"source":["%%time\n","X_train, X_test, y_train, y_test = train_test_split(X_scaler, y, test_size=0.25, random_state=100)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")"]},{"cell_type":"code","execution_count":53,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.11584328]\n"," [-1.09234206]\n"," [-0.4442111 ]\n"," ...\n"," [ 0.31962314]\n"," [ 0.57481565]\n"," [ 0.51973084]]\n","-1.342010788829061\n","0.4421888877678279\n","1536\n","Accuracy: 0.7344040968342644\n","Macro-F1 score: 0.7035967765411122\n","CPU times: total: 12min 19s\n","Wall time: 1min 47s\n"]},{"data":{"text/plain":["0.7035967765411122"]},"execution_count":53,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1 #best: 0.1 (btw 0.10 and 0.16)\n","C = 100 #best: 100\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model8 = score(y_test, predict(X_test, w, b))\n","Model8"]},{"cell_type":"markdown","metadata":{"cell_id":"00068-89a646ec-2454-4320-b8c4-f5548f796b36","deepnote_cell_height":153.1875,"deepnote_cell_type":"markdown"},"source":["## Exporting Prediction\n","Prediction made by your Logistic Regression on the Test set. Note that you are welcome to submit your predicted labels to Kaggle but you will need to submit the final prediction output in the final project submission. Please label the file as \"LogRed_Prediction.csv\"."]},{"cell_type":"code","execution_count":54,"metadata":{"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 5001 columns</p>\n","</div>"],"text/plain":["         id    0    1    2    3    4    5    6    7    8  ...  4990  4991  \\\n","0     17185  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1     17186  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2     17187  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3     17188  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4     17189  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...     ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","4291  21476  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4292  21477  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4293  21478  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4294  21479  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4295  21480  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","      4992  4993  4994  4995  4996  4997  4998  4999  \n","0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...    ...   ...   ...   ...   ...   ...   ...   ...  \n","4291   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4292   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4293   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4294   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4295   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[4296 rows x 5001 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 3.19 s\n","Wall time: 7.3 s\n"]}],"source":["%%time\n","df_test = pd.read_csv(r\"./source/test_tfidf_features.csv\")\n","display(df_test)"]},{"cell_type":"code","execution_count":55,"metadata":{"cell_id":"af6ddade41a94fc8afc67d51267e261b","deepnote_cell_height":634.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":5113,"execution_start":1658684087188,"source_hash":"68d91bc9","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 0 ns\n","Wall time: 0 ns\n"]}],"source":["%%time\n","# df_test = pd.read_csv(\"/work/50007-2022/test_tfidf_features.csv\")\n","# display(df_test)"]},{"cell_type":"code","execution_count":56,"metadata":{"cell_id":"b98d59a7657446668ec90741a9492258","deepnote_cell_height":172.1875,"deepnote_cell_type":"code","deepnote_output_heights":[21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":174,"execution_start":1658684807590,"source_hash":"341533dd","tags":[],"trusted":true},"outputs":[{"data":{"text/plain":["array([1, 0, 1, ..., 1, 0, 0])"]},"metadata":{},"output_type":"display_data"}],"source":["features = df_test.iloc[:,1:]\n","\n","results = predict(features, w, b)\n","display(results)"]},{"cell_type":"code","execution_count":57,"metadata":{"cell_id":"ebef1c3a31934cdda95c464e20cd5788","deepnote_cell_height":148,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":97,"execution_start":1658684807669,"source_hash":"774f40c6","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["(4296, 1) (4296, 1)\n"]}],"source":["df_ids = df_test.iloc[:, 0].to_frame()\n","df_results = pd.DataFrame(results)\n","print(df_results.shape, df_ids.shape)"]},{"cell_type":"code","execution_count":58,"metadata":{"cell_id":"5e35eb9faffc4389a0b402260b3b6d95","deepnote_cell_height":636,"deepnote_cell_type":"code","deepnote_output_heights":[21.1875],"deepnote_table_invalid":false,"deepnote_table_loading":false,"deepnote_table_state":{"filters":[],"pageIndex":0,"pageSize":10,"sortBy":[]},"deepnote_to_be_reexecuted":false,"execution_millis":65,"execution_start":1658684807702,"source_hash":"510702b8","tags":[],"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 2 columns</p>\n","</div>"],"text/plain":["         id  label\n","0     17185      1\n","1     17186      0\n","2     17187      1\n","3     17188      0\n","4     17189      0\n","...     ...    ...\n","4291  21476      0\n","4292  21477      0\n","4293  21478      1\n","4294  21479      0\n","4295  21480      0\n","\n","[4296 rows x 2 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["df_submission = pd.concat([df_ids, df_results], axis =1)\n","df_submission = df_submission.rename(columns={0: 'label'})\n","display(df_submission)"]},{"cell_type":"code","execution_count":59,"metadata":{"cell_id":"e3da92dbbe6c42f28d0802f31f076964","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":23,"execution_start":1658684807735,"source_hash":"21d4263a","tags":[],"trusted":true},"outputs":[],"source":["# Write to csv\n","# df_submission.to_csv(\"LogRed_Prediction.csv\", index=False)"]},{"cell_type":"markdown","metadata":{"cell_id":"00053-672d396b-4f1d-48be-b852-56817a09cd49","deepnote_cell_height":399.1875,"deepnote_cell_type":"markdown"},"source":["# Task 2: Apply Dimension Reduction Techniques\n","\n","Dimension reduction is the transformation of data from a high-dimensional space into a low-dimensional space so that the low-dimensional representation retains some meaningful properties of the original data. \\\n","The train dataset contains 5000 TD-IDF features. In this task, you are to apply PCA to reduce the dimension of features.\n","\n","## Key Task Deliverables\n","\n","2a. Code implementation of PCA on the train and test sets. Note that you are allowed to use the sklearn package for this task. \\\n","2b. Report the Macro F1 scores for applying 2000, 1000, 500, and 100 components on the test set. Note that you will have to submit your predicted labels to Kaggle to retrieve the Macro F1 scores for the test set and report the results in your final report. \\\n","Use KNN as the machine learning model for your training and prediction (You are also allowed to use the sklearn package for KNN implementation) (set n_neighbors=2)."]},{"cell_type":"markdown","metadata":{},"source":["Import Relevant Packages for Task 2"]},{"cell_type":"code","execution_count":1,"metadata":{"cell_id":"00054-b8072742-0be5-4e46-b68e-f2ede4fc6333","deepnote_cell_height":171,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658689638099,"source_hash":"b9067867","trusted":true},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","\n","from sklearn.decomposition import PCA, IncrementalPCA, KernelPCA, TruncatedSVD, MiniBatchSparsePCA\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import f1_score\n","from sklearn.neighbors import KNeighborsClassifier as KNN\n","from sklearn.preprocessing import RobustScaler as Scaler"]},{"cell_type":"markdown","metadata":{},"source":["Define Functions to Evaluate the Model Performance from adopting Dimensionality Reduction. This will be done simultaneously to observe any improvement in Model Performance from using Dimesionality Reduction Techniques"]},{"cell_type":"code","execution_count":2,"metadata":{"trusted":true},"outputs":[],"source":["def sigmoid(z):\n","    result = 1/(1 + np.exp(-z))\n","#     print(f\"sigmoid: {result}\")\n","    return result"]},{"cell_type":"code","execution_count":3,"metadata":{"trusted":true},"outputs":[],"source":["def loss(y, X, w, b, lmb):\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    m = np.shape(y)[0]\n","    \n","    loss = -1 * np.where(y == 1, np.log(y_hat), np.log(1 - y_hat)).mean()\n","    reg = lmb * np.sum(w**2) / (2 * m)\n","    error = loss + reg\n","    \n","#     print(f\"training loss = {loss}, regularisation term = {reg}, training error = {error}\")\n","    return error"]},{"cell_type":"code","execution_count":4,"metadata":{"trusted":true},"outputs":[],"source":["def gradients(y, X, w, b, lmb):\n","    # m - number of training examples\n","    m = np.shape(X)[0]\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    \n","    dw = (1 / m) * (np.dot(X.T, (y_hat - y)) + lmb * w)\n","    db = (1 / m) * np.sum((y_hat - y))\n","    \n","#     print(f\"dw: {dw}, db: {db}\")\n","    return dw, db"]},{"cell_type":"code","execution_count":5,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train3(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":6,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train4(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":7,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train5(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","\n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":8,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train6(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":9,"metadata":{"trusted":true},"outputs":[],"source":["def predict(X, w, b):\n","    y_pred = sigmoid(np.dot(X, w) + b)\n","    pred_labels = np.array([1 if i >= 0.5 else 0 for i in y_pred])\n","    return pred_labels"]},{"cell_type":"code","execution_count":10,"metadata":{"trusted":true},"outputs":[],"source":["def score(y, y_hat):\n","    accuracy = np.sum(y == y_hat) / np.shape(y)[0]\n","    f1score = f1_score(y, y_hat, average='macro')\n","\n","    print(f\"Accuracy: {accuracy}\")\n","    print(f\"Macro-F1 score: {f1score}\")\n","\n","    # Return Macro-F1 score of the model\n","    return f1score"]},{"cell_type":"markdown","metadata":{},"source":["Initialise Training Data for Task 2."]},{"cell_type":"code","execution_count":11,"metadata":{"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>17180</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>17181</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>17182</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>17183</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>17184</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5002 columns</p>\n","</div>"],"text/plain":["          id  label    0    1    2    3    4    5    6    7  ...  4990  4991  \\\n","0          1      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1          2      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2          3      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3          4      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4          5      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...      ...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  17180      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  17181      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  17182      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  17183      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  17184      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5002 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 17 s\n","Wall time: 17.1 s\n"]}],"source":["%%time\n","df_train = pd.read_csv(r\"./source/train_tfidf_features.csv\")\n","display(df_train)"]},{"cell_type":"code","execution_count":12,"metadata":{"cell_id":"00055-91cb2cc9-7846-432c-b255-ecc09e53f6c2","deepnote_cell_height":716,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":42445,"execution_start":1658689639841,"source_hash":"50b848d7","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 0 ns\n","Wall time: 0 ns\n"]}],"source":["%%time\n","# df_train = pd.read_csv(\"/work/50007-2022/train_tfidf_features.csv\")\n","# display(df_train)"]},{"cell_type":"code","execution_count":13,"metadata":{"cell_id":"9c9a773b88434f6db5fcea744c70d847","deepnote_cell_height":117,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":6,"execution_start":1658689631971,"source_hash":"db08c607","tags":[],"trusted":true},"outputs":[],"source":["# Define features and label\n","df_features = df_train.iloc[:,2:]\n","df_label = df_train.iloc[:, 1]"]},{"cell_type":"code","execution_count":14,"metadata":{"cell_id":"2e1a6b992de1448c8ab6e1b63c8d16af","deepnote_cell_height":616.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":2024,"execution_start":1658688637833,"source_hash":"55cf74dd","tags":[],"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5000 columns</p>\n","</div>"],"text/plain":["         0    1    2    3    4    5    6    7    8    9  ...  4990  4991  \\\n","0      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5000 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["display(df_features)"]},{"cell_type":"code","execution_count":15,"metadata":{"cell_id":"969eabaf5a124d86887172291e438e29","deepnote_cell_height":600,"deepnote_cell_type":"code","deepnote_output_heights":[232.390625],"deepnote_to_be_reexecuted":true,"execution_millis":26,"execution_start":1658688639858,"source_hash":"ef276ad2","tags":[],"trusted":true},"outputs":[{"data":{"text/plain":["0        1\n","1        0\n","2        1\n","3        0\n","4        1\n","        ..\n","17179    0\n","17180    0\n","17181    1\n","17182    1\n","17183    0\n","Name: label, Length: 17184, dtype: int64"]},"metadata":{},"output_type":"display_data"}],"source":["display(df_label)"]},{"cell_type":"markdown","metadata":{},"source":["## Initializing the Control Model for Evaluation"]},{"cell_type":"code","execution_count":80,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 5000), y_train: (12888,)\n","X_test: (4296, 5000), y_test: (4296,)\n","CPU times: total: 422 ms\n","Wall time: 423 ms\n"]}],"source":["%%time\n","# scaler = Scaler(with_centering=False).fit(df_features)\n","# X_scaler = scaler.transform(df_features)\n","X_train, X_test, y_train, y_test = train_test_split(df_features, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")"]},{"cell_type":"code","execution_count":81,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.633147113594041\n","Macro-F1 score: 0.5974068478397375\n","CPU times: total: 53.7 s\n","Wall time: 8.28 s\n"]},{"data":{"text/plain":["0.5974068478397375"]},"execution_count":81,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = KNN(n_neighbors=2, weights='distance', metric='cosine', n_jobs=-1)\n","knn.fit(X_train, y_train)\n","score(y_test, knn.predict(X_test))"]},{"cell_type":"markdown","metadata":{},"source":["## Applying Principal Component Analysis (PCA)"]},{"cell_type":"code","execution_count":82,"metadata":{"trusted":true},"outputs":[],"source":["n_components = 1000\n","random_state = 100"]},{"cell_type":"code","execution_count":83,"metadata":{"cell_id":"00056-8acc8aa1-45e7-4e0b-8a3c-4562d28d4d7d","deepnote_cell_height":292,"deepnote_cell_type":"code","deepnote_output_heights":[194],"deepnote_to_be_reexecuted":true,"execution_millis":147893,"execution_start":1658687140783,"source_hash":"9be852a0","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 3min 7s\n","Wall time: 26 s\n"]}],"source":["%%time\n","pca = PCA(n_components=n_components, random_state=random_state)\n","pca.fit(df_features.to_numpy()) #df_features.to_numpy()\n","# print(pca.explained_variance_ratio_)\n","X_pca = pca.transform(df_features.to_numpy()) #df_features.to_numpy()\n","\n","# plt.figure(figsize=(8,6))\n","# plt.scatter(x_pca[:,0], x_3d[:,1], c=y_label) # show plotting of labels visually\n","# plt.show()"]},{"cell_type":"code","execution_count":84,"metadata":{"cell_id":"00057-9ee82dfa-b05d-4471-91b1-da119e05dea0","deepnote_cell_height":484,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":1,"execution_start":1658687288706,"source_hash":"e6cf271d","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["0.647878600154715\n","[[-0.08319779 -0.01604783 -0.0105949  ...  0.03051965  0.00343086\n","  -0.04443635]\n"," [-0.06842061 -0.04364866 -0.01844324 ...  0.03397885  0.02797469\n","   0.03453071]\n"," [-0.08017058 -0.04464213 -0.01534231 ...  0.00096258 -0.00756368\n","  -0.00219514]\n"," ...\n"," [ 0.00207858 -0.04178633 -0.01688574 ...  0.01650203 -0.03759045\n","  -0.02313974]\n"," [ 0.09135488 -0.05590327 -0.00847336 ...  0.00151281  0.00411392\n","  -0.0050238 ]\n"," [-0.04061166  0.09370788 -0.03998554 ...  0.00149003 -0.00706109\n","  -0.01527123]]\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","print(np.sum(pca.explained_variance_ratio_))\n","print(X_pca)\n","print(X_pca.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":52,"metadata":{"cell_id":"00058-60ad6fff-42a2-4fe9-bba2-6e4d2a9e6a93","deepnote_cell_height":186,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":625,"execution_start":1658688674821,"source_hash":"11567c9b","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 46.9 ms\n","Wall time: 48 ms\n"]}],"source":["%%time\n","X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")"]},{"cell_type":"code","execution_count":53,"metadata":{"cell_id":"60773b9298624ba5b5517c858c3670aa","deepnote_cell_height":251,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":6962,"execution_start":1658689289370,"source_hash":"d310c324","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6494413407821229\n","Macro-F1 score: 0.6184697690217391\n","CPU times: total: 13.5 s\n","Wall time: 2.61 s\n"]},{"data":{"text/plain":["0.6184697690217391"]},"execution_count":53,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = KNN(n_neighbors=2, weights='distance', metric='cosine', n_jobs=-1) #neighbors=16\n","knn.fit(X_pcatrain, y_pcatrain)\n","score(y_pcatest, knn.predict(X_pcatest))"]},{"cell_type":"code","execution_count":17,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.654562383612663\n","Macro-F1 score: 0.5653831627887951\n","Model6: 0.5653831627887951\n","\n","[LibLinear]Accuracy: 0.6589851024208566\n","Macro-F1 score: 0.5755167185767787\n","SVC: 0.5755167185767787 \n","\n","Accuracy: 0.6536312849162011\n","Macro-F1 score: 0.5950471503883905\n","KNN: 0.5950471503883905 \n","\n","n_components = 500\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.6964618249534451\n","Macro-F1 score: 0.6491201170120733\n","Model6: 0.6491201170120733\n","\n","[LibLinear]Accuracy: 0.6934357541899442\n","Macro-F1 score: 0.6543271395283521\n","SVC: 0.6543271395283521 \n","\n","Accuracy: 0.6841247672253259\n","Macro-F1 score: 0.6304685257152912\n","KNN: 0.6304685257152912 \n","\n","n_components = 1000\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.7134543761638734\n","Macro-F1 score: 0.6776924223184565\n","Model6: 0.6776924223184565\n","\n","[LibLinear]Accuracy: 0.7029795158286778\n","Macro-F1 score: 0.6735437110026278\n","SVC: 0.6735437110026278 \n","\n","Accuracy: 0.6894785847299814\n","Macro-F1 score: 0.6331016609730239\n","KNN: 0.6331016609730239 \n","\n","n_components = 1500\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.720437616387337\n","Macro-F1 score: 0.6879056526472813\n","Model6: 0.6879056526472813\n","\n","[LibLinear]Accuracy: 0.7048417132216015\n","Macro-F1 score: 0.6794089473829175\n","SVC: 0.6794089473829175 \n","\n","Accuracy: 0.6915735567970205\n","Macro-F1 score: 0.6314628745882833\n","KNN: 0.6314628745882833 \n","\n","n_components = 2000\n","X_train: (12888, 2000), y_train: (12888,)\n","X_test: (4296, 2000), y_test: (4296,)\n","Accuracy: 0.7250931098696461\n","Macro-F1 score: 0.6962892047290179\n","Model6: 0.6962892047290179\n","\n","[LibLinear]Accuracy: 0.7055400372439479\n","Macro-F1 score: 0.6824255719631482\n","SVC: 0.6824255719631482 \n","\n","Accuracy: 0.6983240223463687\n","Macro-F1 score: 0.6373118262478754\n","KNN: 0.6373118262478754 \n","\n","n_components = 2500\n","X_train: (12888, 2500), y_train: (12888,)\n","X_test: (4296, 2500), y_test: (4296,)\n","Accuracy: 0.7267225325884544\n","Macro-F1 score: 0.6958848522737797\n","Model6: 0.6958848522737797\n","\n","[LibLinear]Accuracy: 0.6973929236499069\n","Macro-F1 score: 0.674122193207588\n","SVC: 0.674122193207588 \n","\n","Accuracy: 0.6906424581005587\n","Macro-F1 score: 0.6259145851852614\n","KNN: 0.6259145851852614 \n","\n","n_components = 3000\n","X_train: (12888, 3000), y_train: (12888,)\n","X_test: (4296, 3000), y_test: (4296,)\n","Accuracy: 0.7274208566108007\n","Macro-F1 score: 0.6975050833067005\n","Model6: 0.6975050833067005\n","\n","[LibLinear]Accuracy: 0.6878491620111732\n","Macro-F1 score: 0.6648668402444189\n","SVC: 0.6648668402444189 \n","\n","Accuracy: 0.696927374301676\n","Macro-F1 score: 0.6323913537250152\n","KNN: 0.6323913537250152 \n","\n","CPU times: total: 40min 48s\n","Wall time: 7min 41s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500, 2000, 2500, 3000] #added 2 more n_values\n","random_state = 100\n","\n","for n_components in comps:\n","    # pca = PCA(n_components=n_components, random_state=random_state)\n","    pca = Nystroem(kernel='cosine', gamma=0.3, coef0=0, n_components=n_components, random_state=100, n_jobs=-1)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","#     print(pca.explained_variance_ratio_)\n","    # print(np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    # w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    # Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    # print(f\"Model3: {Model3}\\n\")\n","    \n","    # w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    # Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    # print(f\"Model4: {Model4}\\n\")\n","    \n","    # w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    # Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    # print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","\n","    svc = LinearSVC(C=3, class_weight=None, verbose=100, random_state=100)\n","    svc.fit(X_pcatrain, y_pcatrain)\n","    svcScore = score(y_pcatest, svc.predict(X_pcatest))\n","    print(f\"SVC: {svcScore} \\n\")\n","    \n","    knn = KNN(n_neighbors=16, weights='distance', metric='cosine', n_jobs=-1) # metric='minkowski, cosine'\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")"]},{"cell_type":"markdown","metadata":{},"source":["## Test Set Predictions using PCA\n","\n","As per the grading rubric - \"Perfectly implemented the PCA and KNN. Implemented model is able to run on test sets with reduced components and performed detail analysis of the reduced components\", we shall predict the labels of the Test dataset.\n","\n","We shall first implement a simple pipeline to generate our predictions using 100, 500, 1000 and 2000 components respectively. We will then submit the results to Kaggle and report the scores."]},{"cell_type":"code","execution_count":106,"metadata":{"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 5001 columns</p>\n","</div>"],"text/plain":["         id    0    1    2    3    4    5    6    7    8  ...  4990  4991  \\\n","0     17185  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1     17186  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2     17187  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3     17188  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4     17189  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...     ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","4291  21476  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4292  21477  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4293  21478  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4294  21479  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4295  21480  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","      4992  4993  4994  4995  4996  4997  4998  4999  \n","0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...    ...   ...   ...   ...   ...   ...   ...   ...  \n","4291   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4292   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4293   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4294   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4295   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[4296 rows x 5001 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 3.31 s\n","Wall time: 3.32 s\n"]}],"source":["%%time\n","df_test = pd.read_csv(r\"./source/test_tfidf_features.csv\")\n","display(df_test)"]},{"cell_type":"code","execution_count":107,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 0 ns\n","Wall time: 0 ns\n"]}],"source":["%%time\n","# df_test = pd.read_csv(\"/work/50007-2022/test_tfidf_features.csv\")\n","# display(df_test)"]},{"cell_type":"code","execution_count":108,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","0.20904054096604335\n","(4296, 1) (4296, 1)\n","n_components = 500\n","0.483136414608792\n","(4296, 1) (4296, 1)\n","n_components = 1000\n","0.647878600154715\n","(4296, 1) (4296, 1)\n","n_components = 1500\n","0.7515722940280163\n","(4296, 1) (4296, 1)\n","n_components = 2000\n","0.8247068485234131\n","(4296, 1) (4296, 1)\n","n_components = 2500\n","0.8790757792783456\n","(4296, 1) (4296, 1)\n","n_components = 3000\n","0.9203410112108971\n","(4296, 1) (4296, 1)\n","CPU times: total: 38min 7s\n","Wall time: 4min 15s\n"]}],"source":["%%time\n","comps = [100, 500, 1000, 1500, 2000, 2500, 3000] #added 2 more n_values\n","random_state = 100\n","\n","for n_components in comps:\n","    pca = PCA(n_components=n_components, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","#     print(pca.explained_variance_ratio_)\n","    print(np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pca, df_label)\n","    \n","    features = df_test.iloc[:,1:]\n","    \n","    X_test = pca.transform(features.to_numpy())\n","    results = knn.predict(X_test)\n","#     display(results)\n","    \n","    df_ids = df_test.iloc[:, 0].to_frame()\n","    df_results = pd.DataFrame(results)\n","    print(df_results.shape, df_ids.shape)\n","    \n","    df_submission = pd.concat([df_ids, df_results], axis =1)\n","    df_submission = df_submission.rename(columns={0: 'label'})\n","#     display(df_submission)\n","    \n","#     df_submission.to_csv(f\"KNN_Prediction_PCA_{n_components}.csv\", index=False)"]},{"cell_type":"markdown","metadata":{},"source":["Note: Compare Computation Time, F1-Score and Cross Validation Improvement e.g. x% increase in score vs y% increase in computational time"]},{"cell_type":"markdown","metadata":{},"source":["Based on our submissions to Kaggle, we have attained the following scores for the test set:\n","\n","***PCA using n components -***\n","* n = 100: public score = 0.54753\n","* n = 500: public score = 0.57102\n","* n = 1000: public score = 0.57573\n","* n = 2000: public score = 0.49821"]},{"cell_type":"code","execution_count":109,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1000, random_state=100, batch_size=128\n"]}],"source":["n_components = 1000\n","random_state = 100\n","batch_size = 128\n","\n","print(f\"n_components={n_components}, random_state={random_state}, batch_size={batch_size}\")"]},{"cell_type":"code","execution_count":110,"metadata":{"cell_id":"1265bfcff6824a0f9b9e567a69bd2e8a","deepnote_cell_height":198.1875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":14,"execution_start":1658689263250,"source_hash":"e0c51826","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 28min 19s\n","Wall time: 6min 6s\n"]}],"source":["%%time\n","sparse = MiniBatchSparsePCA(n_components=n_components, batch_size=batch_size, random_state=random_state, n_jobs=-1)\n","sparse.fit(df_features.to_numpy())\n","X_sparse = sparse.transform(df_features.to_numpy())"]},{"cell_type":"code","execution_count":111,"metadata":{"cell_id":"9a5f2d545d8d45348b4ff5806b7f4d97","deepnote_cell_height":138,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"91556f9f","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-6.30100126e-02  0.00000000e+00  0.00000000e+00 ... -6.07939000e-05\n","  -7.58605898e-05 -1.72552829e-05]\n"," [-6.30100126e-02  0.00000000e+00  0.00000000e+00 ... -6.07939000e-05\n","  -7.58605898e-05 -1.72552829e-05]\n"," [-6.30100126e-02  0.00000000e+00  0.00000000e+00 ... -6.07939000e-05\n","  -7.58605898e-05 -1.72552829e-05]\n"," ...\n"," [ 1.06875687e-02  0.00000000e+00  0.00000000e+00 ... -6.07939000e-05\n","  -7.58605898e-05 -1.72552829e-05]\n"," [ 1.12089168e-01  0.00000000e+00  0.00000000e+00 ... -6.07939000e-05\n","  -7.58605898e-05 -1.72552829e-05]\n"," [-5.75933863e-02  0.00000000e+00  0.00000000e+00 ... -6.07939000e-05\n","  -7.58605898e-05 -1.72552829e-05]]\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","print(X_sparse)\n","print(X_sparse.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":112,"metadata":{"cell_id":"af7da6cf66904e78ac25ebd382d2134f","deepnote_cell_height":120,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"758f95a8","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 31.2 ms\n","Wall time: 38 ms\n"]}],"source":["%%time\n","X_sparsetrain, X_sparsetest, y_sparsetrain, y_sparsetest = train_test_split(X_sparse, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_sparsetrain.shape}, y_train: {y_sparsetrain.shape}\")\n","print(f\"X_test: {X_sparsetest.shape}, y_test: {y_sparsetest.shape}\")"]},{"cell_type":"code","execution_count":113,"metadata":{"cell_id":"103204f2bd014ab9ab334767b121b278","deepnote_cell_height":300,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"c11dcd88","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6333798882681564\n","Macro-F1 score: 0.4431337982526638\n","CPU times: total: 2min 18s\n","Wall time: 22 s\n"]},{"data":{"text/plain":["0.4431337982526638"]},"execution_count":113,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_sparsetrain, y_sparsetrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# print(w)\n","# print(b)\n","# print(min(l))\n","# print(l.index(min(l)))\n","score(y_sparsetest.values,  predict(X_sparsetest, w, b))"]},{"cell_type":"code","execution_count":114,"metadata":{"cell_id":"f2907f04e9924fdc8a5fabe8bc127368","deepnote_cell_height":102,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"d310c324","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.619413407821229\n","Macro-F1 score: 0.4904892353994485\n","CPU times: total: 11 s\n","Wall time: 1.04 s\n"]},{"data":{"text/plain":["0.4904892353994485"]},"execution_count":114,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = KNN(n_neighbors=2)\n","knn.fit(X_sparsetrain, y_sparsetrain)\n","score(y_sparsetest, knn.predict(X_sparsetest))"]},{"cell_type":"code","execution_count":115,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.6259310986964618\n","Macro-F1 score: 0.3890550886279467\n","Model3: 0.3890550886279467\n","\n","Accuracy: 0.6373370577281192\n","Macro-F1 score: 0.4493719559087413\n","Model4: 0.4493719559087413\n","\n","Accuracy: 0.6378026070763501\n","Macro-F1 score: 0.450532454261015\n","Model5: 0.450532454261015\n","\n","Accuracy: 0.6371042830540037\n","Macro-F1 score: 0.44879085173073197\n","Model6: 0.44879085173073197\n","\n","Accuracy: 0.6298882681564246\n","Macro-F1 score: 0.4347854868498573\n","KNN: 0.4347854868498573 \n","\n","n_components = 500\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.625\n","Macro-F1 score: 0.3921253963980858\n","Model3: 0.3921253963980858\n","\n","Accuracy: 0.6454841713221602\n","Macro-F1 score: 0.49791883051481517\n","Model4: 0.49791883051481517\n","\n","Accuracy: 0.6454841713221602\n","Macro-F1 score: 0.49899194343458414\n","Model5: 0.49899194343458414\n","\n","Accuracy: 0.6454841713221602\n","Macro-F1 score: 0.49791883051481517\n","Model6: 0.49791883051481517\n","\n","Accuracy: 0.6359404096834265\n","Macro-F1 score: 0.5127674978767247\n","KNN: 0.5127674978767247 \n","\n","n_components = 1000\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.6231378026070763\n","Macro-F1 score: 0.38853534094760345\n","Model3: 0.38853534094760345\n","\n","Accuracy: 0.633147113594041\n","Macro-F1 score: 0.4430103995585214\n","Model4: 0.4430103995585214\n","\n","Accuracy: 0.633147113594041\n","Macro-F1 score: 0.4430103995585214\n","Model5: 0.4430103995585214\n","\n","Accuracy: 0.6333798882681564\n","Macro-F1 score: 0.4431337982526638\n","Model6: 0.4431337982526638\n","\n","Accuracy: 0.619413407821229\n","Macro-F1 score: 0.4904892353994485\n","KNN: 0.4904892353994485 \n","\n","n_components = 1500\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.6243016759776536\n","Macro-F1 score: 0.38899300849660645\n","Model3: 0.38899300849660645\n","\n","Accuracy: 0.6359404096834265\n","Macro-F1 score: 0.45441559213494426\n","Model4: 0.45441559213494426\n","\n","Accuracy: 0.6359404096834265\n","Macro-F1 score: 0.45441559213494426\n","Model5: 0.45441559213494426\n","\n","Accuracy: 0.6359404096834265\n","Macro-F1 score: 0.45441559213494426\n","Model6: 0.45441559213494426\n","\n","Accuracy: 0.6229050279329609\n","Macro-F1 score: 0.4865270455118926\n","KNN: 0.4865270455118926 \n","\n","CPU times: total: 1h 48min 3s\n","Wall time: 27min 42s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500] # removed 2000 due to long computation time\n","\n","for n_components in comps:\n","    pca = MiniBatchSparsePCA(n_components=n_components, batch_size=batch_size, random_state=random_state, n_jobs=-1)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","#     print(pca.explained_variance_ratio_)\n","#     print(np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")"]},{"cell_type":"code","execution_count":116,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1000, random_state=100\n"]}],"source":["n_components = 1000\n","random_state = 100\n","\n","print(f\"n_components={n_components}, random_state={random_state}\")"]},{"cell_type":"code","execution_count":117,"metadata":{"cell_id":"65b25c6d510f4ecfa805b96d6de5ffd0","deepnote_cell_height":166,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":187050,"execution_start":1658687820350,"source_hash":"db4df06f","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 3min 47s\n","Wall time: 26.9 s\n"]}],"source":["%%time\n","svd = TruncatedSVD(n_components=n_components, random_state=random_state)\n","svd.fit(df_features.to_numpy())\n","# print(svd.explained_variance_ratio_)\n","X_svd = svd.transform(df_features.to_numpy())"]},{"cell_type":"code","execution_count":118,"metadata":{"cell_id":"9d2894b2c7f540b1b173e4b8f26a79cf","deepnote_cell_height":484,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":30,"execution_start":1658688007408,"source_hash":"fbc03bf3","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["0.650073164448684\n","[[ 0.02453787  0.01323197  0.01108671 ... -0.02287    -0.0363188\n","   0.00216041]\n"," [ 0.03147557 -0.0160131   0.00925504 ...  0.00301141 -0.01426135\n","  -0.01820294]\n"," [ 0.04311033 -0.00476541  0.02148526 ... -0.00791381 -0.0038874\n","  -0.00240682]\n"," ...\n"," [ 0.07860928 -0.03656004 -0.00391902 ... -0.03735932 -0.01759265\n","  -0.01749524]\n"," [ 0.15991347 -0.06946641 -0.00483063 ... -0.00601389  0.00057129\n","   0.00433898]\n"," [ 0.06143827  0.09825494 -0.04970419 ... -0.01875616 -0.00190562\n","  -0.00171344]]\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","print(np.sum(svd.explained_variance_ratio_))\n","print(X_svd)\n","print(X_svd.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":119,"metadata":{"cell_id":"bb4aac5d0fc7440f8216a9ac85999898","deepnote_cell_height":186,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":122,"execution_start":1658689301898,"source_hash":"8dafaa66","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 46.9 ms\n","Wall time: 51 ms\n"]}],"source":["%%time\n","X_svdtrain, X_svdtest, y_svdtrain, y_svdtest = train_test_split(X_svd, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_svdtrain.shape}, y_train: {y_svdtrain.shape}\")\n","print(f\"X_test: {X_svdtest.shape}, y_test: {y_svdtest.shape}\")"]},{"cell_type":"code","execution_count":120,"metadata":{"cell_id":"063c67f915ab44e49b397ab3e1bb8e5c","deepnote_cell_height":418.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":98527,"execution_start":1658689410287,"source_hash":"c8d685df","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","CPU times: total: 1min 49s\n","Wall time: 23.6 s\n"]},{"data":{"text/plain":["0.6836444589405164"]},"execution_count":120,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_svdtrain, y_svdtrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# print(w)\n","# print(b)\n","# print(min(l))\n","# print(l.index(min(l)))\n","score(y_svdtest.values,  predict(X_svdtest, w, b))"]},{"cell_type":"code","execution_count":121,"metadata":{"cell_id":"7b27c9b3e5f5470fb72b93a9773ba30f","deepnote_cell_height":264.984375,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":6632,"execution_start":1658689542978,"source_hash":"556767a0","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6368715083798883\n","Macro-F1 score: 0.5716805652843039\n","CPU times: total: 9.95 s\n","Wall time: 1.11 s\n"]},{"data":{"text/plain":["0.5716805652843039"]},"execution_count":121,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = KNN(n_neighbors=2)\n","knn.fit(X_svdtrain, y_svdtrain.values)\n","score(y_svdtest, knn.predict(X_svdtest))"]},{"cell_type":"code","execution_count":122,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","0.20770761680685512\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.633147113594041\n","Macro-F1 score: 0.4597114735469538\n","Model3: 0.4597114735469538\n","\n","Accuracy: 0.688780260707635\n","Macro-F1 score: 0.631559639336222\n","Model4: 0.631559639336222\n","\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.633025514433183\n","Model5: 0.633025514433183\n","\n","Accuracy: 0.6894785847299814\n","Macro-F1 score: 0.6329432742672462\n","Model6: 0.6329432742672462\n","\n","Accuracy: 0.6585195530726257\n","Macro-F1 score: 0.565109424225936\n","KNN: 0.565109424225936\n","\n","n_components = 500\n","0.48528147090419305\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.6135940409683427\n","Macro-F1 score: 0.4738487009566307\n","Model3: 0.4738487009566307\n","\n","Accuracy: 0.7171787709497207\n","Macro-F1 score: 0.6779712517078109\n","Model4: 0.6779712517078109\n","\n","Accuracy: 0.7167132216014898\n","Macro-F1 score: 0.6777980209862013\n","Model5: 0.6777980209862013\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model6: 0.6780034986112806\n","\n","Accuracy: 0.6538640595903166\n","Macro-F1 score: 0.5514192419011086\n","KNN: 0.5514192419011086\n","\n","n_components = 1000\n","0.650073164448684\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.6054469273743017\n","Macro-F1 score: 0.48659276617913666\n","Model3: 0.48659276617913666\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","Model4: 0.6836444589405164\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","Model5: 0.6836444589405164\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","Model6: 0.6836444589405164\n","\n","Accuracy: 0.6368715083798883\n","Macro-F1 score: 0.5716805652843039\n","KNN: 0.5716805652843039\n","\n","n_components = 1500\n","0.7535549687544596\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.5998603351955307\n","Macro-F1 score: 0.48584701881984005\n","Model3: 0.48584701881984005\n","\n","Accuracy: 0.720903165735568\n","Macro-F1 score: 0.6870472512776712\n","Model4: 0.6870472512776712\n","\n","Accuracy: 0.720903165735568\n","Macro-F1 score: 0.6870472512776712\n","Model5: 0.6870472512776712\n","\n","Accuracy: 0.720903165735568\n","Macro-F1 score: 0.6870472512776712\n","Model6: 0.6870472512776712\n","\n","Accuracy: 0.5835661080074488\n","Macro-F1 score: 0.5648477343817014\n","KNN: 0.5648477343817014\n","\n","n_components = 2000\n","0.8263684695508671\n","X_train: (12888, 2000), y_train: (12888,)\n","X_test: (4296, 2000), y_test: (4296,)\n","Accuracy: 0.5947392923649907\n","Macro-F1 score: 0.4886192293368272\n","Model3: 0.4886192293368272\n","\n","Accuracy: 0.7216014897579144\n","Macro-F1 score: 0.6889435203589405\n","Model4: 0.6889435203589405\n","\n","Accuracy: 0.7216014897579144\n","Macro-F1 score: 0.6889435203589405\n","Model5: 0.6889435203589405\n","\n","Accuracy: 0.7216014897579144\n","Macro-F1 score: 0.6889435203589405\n","Model6: 0.6889435203589405\n","\n","Accuracy: 0.5223463687150838\n","Macro-F1 score: 0.5213295619146662\n","KNN: 0.5213295619146662\n","\n","n_components = 2500\n","0.8804159851176797\n","X_train: (12888, 2500), y_train: (12888,)\n","X_test: (4296, 2500), y_test: (4296,)\n","Accuracy: 0.5947392923649907\n","Macro-F1 score: 0.4915955941895638\n","Model3: 0.4915955941895638\n","\n","Accuracy: 0.7183426443202979\n","Macro-F1 score: 0.6841218700284756\n","Model4: 0.6841218700284756\n","\n","Accuracy: 0.7183426443202979\n","Macro-F1 score: 0.6841218700284756\n","Model5: 0.6841218700284756\n","\n","Accuracy: 0.7183426443202979\n","Macro-F1 score: 0.6841218700284756\n","Model6: 0.6841218700284756\n","\n","Accuracy: 0.4851024208566108\n","Macro-F1 score: 0.4838925521302222\n","KNN: 0.4838925521302222\n","\n","n_components = 3000\n","0.9213645763699944\n","X_train: (12888, 3000), y_train: (12888,)\n","X_test: (4296, 3000), y_test: (4296,)\n","Accuracy: 0.6031191806331471\n","Macro-F1 score: 0.501319942933531\n","Model3: 0.501319942933531\n","\n","Accuracy: 0.7192737430167597\n","Macro-F1 score: 0.6860245174718075\n","Model4: 0.6860245174718075\n","\n","Accuracy: 0.7192737430167597\n","Macro-F1 score: 0.6860245174718075\n","Model5: 0.6860245174718075\n","\n","Accuracy: 0.7192737430167597\n","Macro-F1 score: 0.6860245174718075\n","Model6: 0.6860245174718075\n","\n","Accuracy: 0.46997206703910616\n","Macro-F1 score: 0.46368867250008516\n","KNN: 0.46368867250008516\n","\n","CPU times: total: 2h 27min 55s\n","Wall time: 20min 42s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500, 2000, 2500, 3000]\n","\n","for n_components in comps:\n","    pca = TruncatedSVD(n_components=n_components, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","#     print(pca.explained_variance_ratio_)\n","    print(np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore}\\n\")"]},{"cell_type":"code","execution_count":123,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1000, batch_size=2000\n"]}],"source":["n_components = 1000\n","batch_size = 2*n_components # must be greater than n_components\n","\n","print(f\"n_components={n_components}, batch_size={batch_size}\")"]},{"cell_type":"code","execution_count":124,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[0.01022275 0.00672865 0.00610278 0.00474011 0.00446933 0.00408268\n"," 0.003797   0.00365259 0.00356391 0.00335548 0.0032464  0.00318154\n"," 0.00303578 0.00297979 0.0029425  0.00285644 0.00277849 0.00251447\n"," 0.00243017 0.00238446 0.00234315 0.0023345  0.00226575 0.00225786\n"," 0.00222997 0.00221648 0.00219955 0.00214889 0.00213189 0.00210605\n"," 0.00207835 0.00206145 0.00202646 0.00201939 0.00200727 0.00198815\n"," 0.00197488 0.00195801 0.001936   0.00190655 0.00188153 0.00187664\n"," 0.00185685 0.00183811 0.00180105 0.00179678 0.00176042 0.00173084\n"," 0.00170405 0.00168805 0.00165099 0.00164993 0.00164429 0.00161694\n"," 0.00160743 0.00158615 0.00157227 0.00155685 0.00155416 0.00154087\n"," 0.00152566 0.0015188  0.00151344 0.00150573 0.00150332 0.00148599\n"," 0.00147808 0.00146831 0.0014584  0.00145364 0.00144314 0.00141747\n"," 0.00140963 0.00140289 0.00139425 0.00138591 0.00137178 0.00136775\n"," 0.00135859 0.00135106 0.00134832 0.00134385 0.00134002 0.00133628\n"," 0.00133346 0.00131965 0.0013124  0.00130912 0.00130031 0.00128732\n"," 0.00128591 0.00127811 0.00127631 0.00126804 0.00126375 0.00125012\n"," 0.00124333 0.00124088 0.001236   0.00123051 0.0012171  0.0012096\n"," 0.00120498 0.00119627 0.00118455 0.00117617 0.00117166 0.00116116\n"," 0.00115542 0.0011517  0.00114468 0.00113935 0.00112928 0.00112752\n"," 0.00112394 0.00112155 0.00111301 0.00110906 0.00110706 0.00109933\n"," 0.00109778 0.00109182 0.00108972 0.00108407 0.0010805  0.00107748\n"," 0.0010723  0.0010644  0.00105921 0.00105742 0.00105102 0.0010473\n"," 0.00104459 0.00104086 0.00103703 0.00103371 0.00102203 0.00101984\n"," 0.00101433 0.0010112  0.00100789 0.00100324 0.00100231 0.00099577\n"," 0.0009951  0.00098996 0.00098702 0.00097912 0.00097378 0.0009718\n"," 0.00097108 0.00096436 0.00095911 0.00095474 0.00095399 0.00095089\n"," 0.0009462  0.00094252 0.00094078 0.00093494 0.00093435 0.00093078\n"," 0.00092655 0.00092206 0.00092029 0.00091692 0.00091353 0.0009113\n"," 0.00090601 0.00090479 0.0009005  0.00089343 0.00089138 0.00088809\n"," 0.00088788 0.00088588 0.0008838  0.00087829 0.00087632 0.00087333\n"," 0.0008714  0.00086808 0.000863   0.00086177 0.0008597  0.00085717\n"," 0.00085473 0.00084938 0.00084849 0.00084236 0.00083922 0.00083801\n"," 0.00083661 0.00083403 0.00083372 0.00082768 0.00082647 0.00082504\n"," 0.00082404 0.00082392 0.00081971 0.00081839 0.00081485 0.00081399\n"," 0.00081028 0.00080893 0.0008074  0.00080451 0.00080199 0.0007998\n"," 0.00079749 0.00079495 0.00079182 0.00079032 0.00078874 0.00078603\n"," 0.0007838  0.00078174 0.00078023 0.00077672 0.00077516 0.00077268\n"," 0.00077141 0.00076925 0.00076562 0.00076454 0.00076347 0.00076196\n"," 0.00075777 0.00075587 0.00075329 0.00075217 0.00074927 0.00074756\n"," 0.00074678 0.00074539 0.0007415  0.00073866 0.00073714 0.00073598\n"," 0.00073514 0.00073297 0.00073062 0.00072953 0.00072689 0.00072457\n"," 0.0007217  0.00072008 0.00071941 0.00071815 0.00071683 0.00071537\n"," 0.00071372 0.0007108  0.0007098  0.0007071  0.00070501 0.00070322\n"," 0.00070211 0.00070119 0.00070008 0.00069707 0.00069595 0.00069415\n"," 0.00069285 0.00069134 0.0006907  0.00068838 0.00068412 0.00068385\n"," 0.00068286 0.00068127 0.00068024 0.00067791 0.00067605 0.00067528\n"," 0.0006726  0.0006722  0.00067146 0.00066595 0.00066516 0.00066361\n"," 0.00066253 0.0006605  0.00065989 0.00065654 0.0006564  0.00065397\n"," 0.00065146 0.00064995 0.00064755 0.0006459  0.00064542 0.00064398\n"," 0.00064319 0.00064105 0.00064036 0.00064013 0.00063817 0.00063555\n"," 0.00063481 0.00063321 0.00063132 0.00062822 0.00062736 0.00062633\n"," 0.00062461 0.0006227  0.00062188 0.0006206  0.00061801 0.0006158\n"," 0.00061544 0.00061509 0.00061246 0.00061059 0.00060962 0.00060876\n"," 0.0006079  0.00060606 0.00060357 0.00060281 0.00060071 0.0006002\n"," 0.00059905 0.00059755 0.00059685 0.00059649 0.00059444 0.00059309\n"," 0.0005927  0.0005906  0.00058958 0.00058862 0.00058752 0.0005862\n"," 0.00058428 0.0005824  0.00057961 0.00057897 0.00057814 0.00057771\n"," 0.00057639 0.00057555 0.00057478 0.00057344 0.00057072 0.0005696\n"," 0.00056865 0.00056772 0.00056699 0.00056634 0.00056412 0.00056321\n"," 0.00056263 0.00055976 0.00055874 0.00055826 0.00055802 0.0005566\n"," 0.00055484 0.00055315 0.00055175 0.00055141 0.00054977 0.00054863\n"," 0.00054629 0.00054533 0.0005437  0.00054258 0.00054113 0.00054015\n"," 0.00053963 0.00053919 0.00053766 0.00053512 0.00053411 0.00053297\n"," 0.00053204 0.0005316  0.00053043 0.00052908 0.00052794 0.00052757\n"," 0.00052586 0.00052495 0.00052447 0.00052332 0.00052193 0.00052093\n"," 0.00051948 0.00051922 0.00051877 0.00051677 0.00051598 0.00051393\n"," 0.00051245 0.00051227 0.00051073 0.00051042 0.00050948 0.00050682\n"," 0.00050625 0.00050557 0.00050513 0.00050377 0.00050274 0.00050215\n"," 0.00050134 0.00050014 0.00049909 0.00049847 0.00049702 0.0004964\n"," 0.00049556 0.00049491 0.00049378 0.0004929  0.00049196 0.000491\n"," 0.00049003 0.00048931 0.00048851 0.00048737 0.00048582 0.0004852\n"," 0.00048433 0.00048363 0.00048213 0.00048149 0.00047993 0.0004793\n"," 0.00047865 0.00047844 0.00047785 0.00047669 0.0004757  0.00047462\n"," 0.00047409 0.00047305 0.0004718  0.00047118 0.00047011 0.00046959\n"," 0.00046943 0.00046787 0.0004676  0.00046591 0.00046486 0.00046407\n"," 0.00046313 0.00046284 0.00046203 0.0004617  0.0004604  0.00045948\n"," 0.00045856 0.0004571  0.00045674 0.00045603 0.00045578 0.00045483\n"," 0.00045325 0.00045298 0.00045182 0.00045177 0.00045044 0.0004495\n"," 0.00044882 0.00044777 0.00044742 0.00044674 0.00044485 0.00044458\n"," 0.00044428 0.00044324 0.00044223 0.00044158 0.0004407  0.0004397\n"," 0.00043908 0.00043862 0.00043796 0.00043667 0.0004356  0.00043455\n"," 0.00043353 0.00043302 0.00043278 0.00043161 0.0004314  0.00043079\n"," 0.00042982 0.00042956 0.00042879 0.00042786 0.00042694 0.00042608\n"," 0.00042535 0.00042401 0.00042364 0.00042349 0.0004228  0.00042191\n"," 0.00042152 0.00042063 0.00042045 0.00041907 0.0004187  0.00041824\n"," 0.00041673 0.00041663 0.00041566 0.0004154  0.00041457 0.00041351\n"," 0.00041287 0.00041249 0.00041186 0.00041105 0.00041067 0.00040987\n"," 0.00040924 0.00040899 0.0004084  0.00040722 0.00040625 0.00040536\n"," 0.00040513 0.00040428 0.0004037  0.00040273 0.00040204 0.00040151\n"," 0.00040087 0.00040058 0.00039984 0.00039948 0.00039943 0.00039809\n"," 0.00039726 0.00039595 0.00039559 0.00039503 0.00039456 0.00039387\n"," 0.00039353 0.00039265 0.00039201 0.00039139 0.00039093 0.00038949\n"," 0.00038913 0.00038881 0.0003881  0.00038733 0.00038691 0.00038651\n"," 0.00038547 0.00038528 0.00038444 0.00038403 0.0003831  0.00038226\n"," 0.00038199 0.00038164 0.00038099 0.00038003 0.00037978 0.00037906\n"," 0.00037888 0.00037843 0.00037816 0.00037727 0.00037629 0.00037566\n"," 0.00037497 0.00037448 0.00037408 0.00037334 0.0003729  0.00037227\n"," 0.00037132 0.00037099 0.0003706  0.00037006 0.00036977 0.00036906\n"," 0.00036842 0.00036781 0.00036738 0.0003671  0.00036649 0.00036597\n"," 0.00036476 0.00036433 0.00036423 0.00036377 0.00036342 0.00036235\n"," 0.00036225 0.00036166 0.000361   0.00036012 0.0003596  0.00035895\n"," 0.00035848 0.0003581  0.00035764 0.00035717 0.00035676 0.00035643\n"," 0.00035597 0.00035468 0.00035457 0.00035361 0.0003532  0.00035266\n"," 0.00035221 0.00035194 0.00035166 0.00035115 0.00035053 0.00034992\n"," 0.00034925 0.00034878 0.00034852 0.00034773 0.00034738 0.00034694\n"," 0.00034621 0.00034591 0.0003452  0.00034432 0.00034383 0.00034332\n"," 0.00034294 0.00034246 0.00034187 0.00034105 0.00034093 0.00034039\n"," 0.00033916 0.00033904 0.00033851 0.00033822 0.00033792 0.00033752\n"," 0.00033729 0.00033659 0.00033597 0.00033576 0.00033538 0.00033482\n"," 0.00033439 0.00033343 0.00033302 0.00033267 0.00033211 0.00033185\n"," 0.00033134 0.00033072 0.00033037 0.00033005 0.00032929 0.0003288\n"," 0.0003281  0.0003278  0.00032713 0.00032678 0.00032594 0.00032563\n"," 0.00032505 0.00032483 0.00032407 0.0003236  0.00032347 0.00032269\n"," 0.00032263 0.00032144 0.0003211  0.00032081 0.00031991 0.00031974\n"," 0.00031902 0.00031876 0.00031846 0.00031806 0.00031742 0.00031723\n"," 0.00031658 0.00031612 0.00031546 0.00031473 0.00031446 0.00031434\n"," 0.00031332 0.00031299 0.0003126  0.00031221 0.00031156 0.00031111\n"," 0.00031075 0.00031026 0.00030981 0.00030957 0.00030892 0.00030868\n"," 0.00030802 0.00030756 0.00030725 0.00030658 0.00030627 0.00030611\n"," 0.00030589 0.00030513 0.00030471 0.00030439 0.00030385 0.00030346\n"," 0.00030228 0.00030198 0.00030163 0.00030114 0.00030044 0.00030024\n"," 0.00029959 0.00029926 0.00029878 0.00029846 0.00029819 0.00029761\n"," 0.00029754 0.00029685 0.00029633 0.00029594 0.00029547 0.0002952\n"," 0.00029469 0.00029419 0.00029373 0.00029354 0.00029313 0.00029263\n"," 0.00029248 0.00029155 0.00029064 0.0002904  0.00029027 0.00028965\n"," 0.00028934 0.00028914 0.00028846 0.00028779 0.00028748 0.00028701\n"," 0.00028666 0.00028657 0.00028611 0.00028584 0.00028512 0.00028491\n"," 0.00028473 0.00028461 0.00028429 0.00028357 0.00028311 0.00028259\n"," 0.00028247 0.00028186 0.00028138 0.00028088 0.00028074 0.00028042\n"," 0.00028015 0.00027944 0.00027875 0.00027858 0.00027817 0.00027773\n"," 0.00027717 0.00027674 0.00027659 0.00027616 0.00027566 0.00027549\n"," 0.00027489 0.0002744  0.00027403 0.00027381 0.00027361 0.0002732\n"," 0.00027282 0.00027231 0.00027132 0.00027067 0.00027009 0.00026998\n"," 0.00026971 0.00026936 0.00026929 0.00026907 0.00026826 0.00026808\n"," 0.00026759 0.00026718 0.00026696 0.00026678 0.00026619 0.00026607\n"," 0.00026549 0.00026518 0.0002644  0.0002643  0.00026425 0.00026371\n"," 0.000263   0.00026237 0.00026219 0.000262   0.00026147 0.00026102\n"," 0.00026037 0.00025994 0.0002597  0.00025934 0.00025903 0.00025859\n"," 0.00025842 0.00025819 0.00025798 0.00025754 0.00025713 0.00025685\n"," 0.00025623 0.00025572 0.00025517 0.00025474 0.00025438 0.0002542\n"," 0.00025373 0.00025351 0.00025279 0.00025239 0.00025211 0.00025194\n"," 0.00025146 0.00025114 0.00025047 0.00025021 0.00024998 0.00024968\n"," 0.00024955 0.00024932 0.00024886 0.00024854 0.000248   0.00024726\n"," 0.00024711 0.00024634 0.00024596 0.00024567 0.00024547 0.00024514\n"," 0.00024473 0.00024437 0.00024387 0.0002437  0.00024299 0.00024278\n"," 0.00024253 0.00024201 0.0002418  0.0002412  0.00024092 0.00024076\n"," 0.00024004 0.0002396  0.00023936 0.0002392  0.00023882 0.00023853\n"," 0.00023806 0.00023766 0.00023708 0.00023657 0.00023622 0.00023601\n"," 0.00023578 0.00023526 0.00023488 0.00023447 0.00023407 0.00023397\n"," 0.00023366 0.00023307 0.0002326  0.00023237 0.00023211 0.0002314\n"," 0.00023115 0.00023085 0.0002304  0.00023021 0.00022963 0.0002295\n"," 0.00022918 0.00022828 0.00022807 0.00022771 0.00022737 0.00022707\n"," 0.00022661 0.00022656 0.0002264  0.00022556 0.00022524 0.00022419\n"," 0.0002239  0.00022386 0.00022321 0.00022286 0.00022272 0.00022211\n"," 0.00022187 0.00022157 0.00022092 0.0002208  0.0002201  0.0002199\n"," 0.00021981 0.0002192  0.00021879 0.0002185  0.00021813 0.00021806\n"," 0.0002174  0.00021713 0.0002164  0.00021607 0.00021593 0.00021539\n"," 0.00021514 0.00021453 0.00021413 0.00021368 0.00021339 0.00021302\n"," 0.00021255 0.000212   0.00021159 0.0002115  0.00021007 0.00020971\n"," 0.00020953 0.00020936 0.00020897 0.00020812 0.0002077  0.00020714\n"," 0.00020687 0.0002067  0.00020621 0.00020603 0.00020522 0.00020479\n"," 0.00020423 0.00020392 0.00020356 0.00020301 0.00020228 0.00020128\n"," 0.00020093 0.00020065 0.00020002 0.00019943 0.00019869 0.00019838\n"," 0.00019807 0.00019689 0.00019643 0.0001959  0.00019572 0.00019469\n"," 0.00019458 0.00019419 0.00019324 0.0001925  0.00019225 0.00019156\n"," 0.00019062 0.00018958 0.00018902 0.00018824 0.00018728 0.00018485\n"," 0.00018383 0.00018297 0.0001804  0.0001695 ]\n","CPU times: total: 20min 25s\n","Wall time: 2min 43s\n"]}],"source":["%%time\n","increment = IncrementalPCA(n_components=n_components, batch_size=batch_size)\n","increment.fit(df_features.to_numpy())\n","print(increment.explained_variance_ratio_)\n","X_increment = increment.transform(df_features.to_numpy())"]},{"cell_type":"code","execution_count":125,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["0.631600694695548\n","[[-0.0832001  -0.01602625 -0.01058633 ...  0.02711227 -0.01133995\n","   0.00187589]\n"," [-0.06841326 -0.04365747 -0.01843916 ... -0.01610998 -0.00508343\n","  -0.0056373 ]\n"," [-0.08016927 -0.04463634 -0.01534796 ... -0.01227815 -0.00147301\n","  -0.00373332]\n"," ...\n"," [ 0.00208556 -0.04178237 -0.01686786 ... -0.01221428  0.01589715\n","   0.00423743]\n"," [ 0.09135843 -0.05587118 -0.00847515 ...  0.0006601   0.00879209\n","   0.00505577]\n"," [-0.04061236  0.09370386 -0.03996926 ...  0.00422785  0.01080164\n","  -0.00838382]]\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","print(np.sum(increment.explained_variance_ratio_))\n","print(X_increment)\n","print(X_increment.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":126,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 31.2 ms\n","Wall time: 38 ms\n"]}],"source":["%%time\n","X_incrementtrain, X_incrementtest, y_incrementtrain, y_incrementtest = train_test_split(X_increment, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_incrementtrain.shape}, y_train: {y_incrementtrain.shape}\")\n","print(f\"X_test: {X_incrementtest.shape}, y_test: {y_incrementtest.shape}\")"]},{"cell_type":"code","execution_count":127,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7195065176908753\n","Macro-F1 score: 0.6847195618177574\n","CPU times: total: 2min 21s\n","Wall time: 21.7 s\n"]},{"data":{"text/plain":["0.6847195618177574"]},"execution_count":127,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_incrementtrain, y_incrementtrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# print(w)\n","# print(b)\n","# print(min(l))\n","# print(l.index(min(l)))\n","score(y_incrementtest.values,  predict(X_incrementtest, w, b))"]},{"cell_type":"code","execution_count":128,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100, batch_size = 4000\n","0.20455528952164242\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.6343109869646183\n","Macro-F1 score: 0.4624685846725459\n","Model3: 0.4624685846725459\n","\n","Accuracy: 0.6908752327746741\n","Macro-F1 score: 0.6369257086999023\n","Model4: 0.6369257086999023\n","\n","Accuracy: 0.6911080074487895\n","Macro-F1 score: 0.6380355527962599\n","Model5: 0.6380355527962599\n","\n","Accuracy: 0.6913407821229051\n","Macro-F1 score: 0.6377775313432503\n","Model6: 0.6377775313432503\n","\n","Accuracy: 0.6613128491620112\n","Macro-F1 score: 0.5686668113488322\n","KNN: 0.5686668113488322 \n","\n","n_components = 500, batch_size = 4000\n","0.4750607106451144\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.6252327746741154\n","Macro-F1 score: 0.4903620199836186\n","Model3: 0.4903620199836186\n","\n","Accuracy: 0.714851024208566\n","Macro-F1 score: 0.6759182424233281\n","Model4: 0.6759182424233281\n","\n","Accuracy: 0.715316573556797\n","Macro-F1 score: 0.6762095149269713\n","Model5: 0.6762095149269713\n","\n","Accuracy: 0.715316573556797\n","Macro-F1 score: 0.6762095149269713\n","Model6: 0.6762095149269713\n","\n","Accuracy: 0.654096834264432\n","Macro-F1 score: 0.5528786189352892\n","KNN: 0.5528786189352892 \n","\n","n_components = 1000, batch_size = 4000\n","0.6365485970168069\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.6212756052141527\n","Macro-F1 score: 0.5031069005987306\n","Model3: 0.5031069005987306\n","\n","Accuracy: 0.7188081936685289\n","Macro-F1 score: 0.6849681226762188\n","Model4: 0.6849681226762188\n","\n","Accuracy: 0.7188081936685289\n","Macro-F1 score: 0.6849681226762188\n","Model5: 0.6849681226762188\n","\n","Accuracy: 0.7188081936685289\n","Macro-F1 score: 0.6849681226762188\n","Model6: 0.6849681226762188\n","\n","Accuracy: 0.6291899441340782\n","Macro-F1 score: 0.5672758840014516\n","KNN: 0.5672758840014516 \n","\n","n_components = 1500, batch_size = 4000\n","0.7401017867402803\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.6145251396648045\n","Macro-F1 score: 0.5053861104954276\n","Model3: 0.5053861104954276\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6907014118132113\n","Model4: 0.6907014118132113\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6907014118132113\n","Model5: 0.6907014118132113\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6907014118132113\n","Model6: 0.6907014118132113\n","\n","Accuracy: 0.5735567970204841\n","Macro-F1 score: 0.5548794834020465\n","KNN: 0.5548794834020465 \n","\n","CPU times: total: 2h 2min 51s\n","Wall time: 16min 23s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500] # removed 2000 due to long computation time\n","\n","for n_components in comps:\n","    batch_size = 4000 # OR 2*n_components\n","    pca = IncrementalPCA(n_components=n_components, batch_size=batch_size)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}, batch_size = {batch_size}\")\n","#     print(pca.explained_variance_ratio_)\n","    print(np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")"]},{"cell_type":"code","execution_count":129,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1000, random_state=100, kernel=cosine, n_jobs=-1\n"]}],"source":["n_components=1000\n","kernel=\"cosine\"\n","n_jobs=-1\n","random_state=100\n","print(f\"n_components={n_components}, random_state={random_state}, kernel={kernel}, n_jobs={n_jobs}\")"]},{"cell_type":"code","execution_count":130,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 1h 8min 13s\n","Wall time: 10min 59s\n"]}],"source":["%%time\n","kern = KernelPCA(n_components=n_components, kernel=kernel, n_jobs=n_jobs, random_state=random_state)\n","kern.fit(df_features.to_numpy())\n","# print(kernel.explained_variance_ratio_)\n","X_kernel = kern.transform(df_features.to_numpy())"]},{"cell_type":"code","execution_count":131,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.08319779 -0.01604783 -0.0105949  ...  0.00179469 -0.01203666\n","  -0.01116068]\n"," [-0.06842061 -0.04364866 -0.01844324 ...  0.00293388  0.01088659\n","  -0.02332199]\n"," [-0.08017058 -0.04464213 -0.01534231 ...  0.00314898 -0.00193481\n","   0.00612507]\n"," ...\n"," [ 0.00207858 -0.04178633 -0.01688574 ...  0.01654121  0.00395149\n","   0.00196094]\n"," [ 0.09135488 -0.05590327 -0.00847336 ...  0.00097483  0.00756974\n","   0.00577514]\n"," [-0.04061166  0.09370788 -0.03998554 ... -0.00583051 -0.012944\n","   0.03945767]]\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","# print(np.sum(kernel.explained_variance_ratio_))\n","print(X_kernel)\n","print(X_kernel.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":132,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 62.5 ms\n","Wall time: 50 ms\n"]}],"source":["%%time\n","X_kerneltrain, X_kerneltest, y_kerneltrain, y_kerneltest = train_test_split(X_kernel, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_kerneltrain.shape}, y_train: {y_kerneltrain.shape}\")\n","print(f\"X_test: {X_kerneltest.shape}, y_test: {y_kerneltest.shape}\")"]},{"cell_type":"code","execution_count":133,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7199720670391061\n","Macro-F1 score: 0.686003205410374\n","CPU times: total: 2min 15s\n","Wall time: 23.6 s\n"]},{"data":{"text/plain":["0.686003205410374"]},"execution_count":133,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_kerneltrain, y_kerneltrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# print(w)\n","# print(b)\n","# print(min(l))\n","# print(l.index(min(l)))\n","score(y_kerneltest.values,  predict(X_kerneltest, w, b))"]},{"cell_type":"code","execution_count":134,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.6291899441340782\n","Macro-F1 score: 0.4591094845338825\n","Model3: 0.4591094845338825\n","\n","Accuracy: 0.6843575418994413\n","Macro-F1 score: 0.6283270398647594\n","Model4: 0.6283270398647594\n","\n","Accuracy: 0.6838919925512105\n","Macro-F1 score: 0.6274612976525208\n","Model5: 0.6274612976525208\n","\n","Accuracy: 0.6848230912476723\n","Macro-F1 score: 0.629190550988352\n","Model6: 0.629190550988352\n","\n","Accuracy: 0.6552607076350093\n","Macro-F1 score: 0.5604755619304248\n","KNN: 0.5604755619304248\n","\n","n_components = 500\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.4881696700131085\n","Model3: 0.4881696700131085\n","\n","Accuracy: 0.7176443202979516\n","Macro-F1 score: 0.6789750953871346\n","Model4: 0.6789750953871346\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model5: 0.6780034986112806\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model6: 0.6780034986112806\n","\n","Accuracy: 0.6524674115456238\n","Macro-F1 score: 0.5456504896637868\n","KNN: 0.5456504896637868\n","\n","n_components = 1000\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.6180167597765364\n","Macro-F1 score: 0.499724057325964\n","Model3: 0.499724057325964\n","\n","Accuracy: 0.7199720670391061\n","Macro-F1 score: 0.686003205410374\n","Model4: 0.686003205410374\n","\n","Accuracy: 0.7199720670391061\n","Macro-F1 score: 0.686003205410374\n","Model5: 0.686003205410374\n","\n","Accuracy: 0.7199720670391061\n","Macro-F1 score: 0.686003205410374\n","Model6: 0.686003205410374\n","\n","Accuracy: 0.6431564245810056\n","Macro-F1 score: 0.5741824982582959\n","KNN: 0.5741824982582959\n","\n","CPU times: total: 3h 16min 10s\n","Wall time: 26min 26s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000] # removed 1500, 2000 due to long computation time\n","kernel = \"cosine\"\n","\n","for n_components in comps:\n","    pca = KernelPCA(n_components=n_components, kernel=kernel, n_jobs=n_jobs, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","#     print(pca.explained_variance_ratio_)\n","#     print(np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore}\\n\")"]},{"cell_type":"code","execution_count":135,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=500, random_state=100, kernels=['linear', 'poly', 'rbf', 'sigmoid', 'cosine'], n_jobs=-1\n","kernel = linear\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.4881696700131085\n","Model3: 0.4881696700131085\n","\n","Accuracy: 0.7176443202979516\n","Macro-F1 score: 0.6789750953871346\n","Model4: 0.6789750953871346\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model5: 0.6780034986112806\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model6: 0.6780034986112806\n","\n","Accuracy: 0.6524674115456238\n","Macro-F1 score: 0.5456504896637868\n","KNN: 0.5456504896637868\n","\n","kernel = poly\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model3: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model4: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model5: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model6: 0.384791636832307\n","\n","Accuracy: 0.6524674115456238\n","Macro-F1 score: 0.5456504896637868\n","KNN: 0.5456504896637868\n","\n","kernel = rbf\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model3: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model4: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model5: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model6: 0.384791636832307\n","\n","Accuracy: 0.6524674115456238\n","Macro-F1 score: 0.5456504896637868\n","KNN: 0.5456504896637868\n","\n","kernel = sigmoid\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model3: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model4: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model5: 0.384791636832307\n","\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model6: 0.384791636832307\n","\n","Accuracy: 0.6524674115456238\n","Macro-F1 score: 0.5456504896637868\n","KNN: 0.5456504896637868\n","\n","kernel = cosine\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.4881696700131085\n","Model3: 0.4881696700131085\n","\n","Accuracy: 0.7176443202979516\n","Macro-F1 score: 0.6789750953871346\n","Model4: 0.6789750953871346\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model5: 0.6780034986112806\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model6: 0.6780034986112806\n","\n","Accuracy: 0.6524674115456238\n","Macro-F1 score: 0.5456504896637868\n","KNN: 0.5456504896637868\n","\n","CPU times: total: 5h 23min 12s\n","Wall time: 44min 38s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","n_components=500\n","Kernels=[\"linear\", \"poly\", \"rbf\", \"sigmoid\", \"cosine\"]\n","n_jobs=-1\n","random_state=100\n","print(f\"n_components={n_components}, random_state={random_state}, kernels={Kernels}, n_jobs={n_jobs}\")\n","\n","for kernel in Kernels:\n","    pca = KernelPCA(n_components=n_components, kernel=kernel, n_jobs=n_jobs, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"kernel = {kernel}\")\n","#     print(pca.explained_variance_ratio_)\n","#     print(np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore}\\n\")"]},{"cell_type":"code","execution_count":136,"metadata":{"cell_id":"00061-90ea847a-5803-4ecf-9e2d-5e5fa71469fb","deepnote_cell_height":297,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"owner_user_id":"6c663559-ea69-4981-bb8f-2bb1a932cbf0","source_hash":"982ea2ff","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["2048 256 0.1 0 0.6 0.999 0.01\n"]}],"source":["# # Comment/ Uncomment to compare with train4's f1 score\n","\n","print(bs, epochs, lr, C, beta_m, beta_v, err)\n","# # w, b, l = train(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","# # print(f1_score(y_pcatest, predict(X_pcatest, w, b), average='macro'))\n","# # w, b, l = train1(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","# # print(f1_score(y_pcatest, predict(X_pcatest, w, b), average='macro'))\n","# # w, b, l = train2(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","# # print(f1_score(y_pcatest, predict(X_pcatest, w, b), average='macro'))\n","# # w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# # print(f1_score(y_pcatest, predict(X_pcatest, w, b), average='macro'))\n","# w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# print(f1_score(y_pcatest, predict(X_pcatest, w, b), average='macro'))\n","# w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# print(f1_score(y_pcatest, predict(X_pcatest, w, b), average='macro'))\n","# w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","# print(f1_score(y_pcatest, predict(X_pcatest, w, b), average='macro'))"]},{"cell_type":"markdown","metadata":{"cell_id":"00062-f73c3f4e-a47c-450e-8c47-b201ff98ced7","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown"},"source":["***Convert to csv for submission using the steps below:***"]},{"cell_type":"code","execution_count":137,"metadata":{"cell_id":"00063-9456e51c-f2b1-4f53-9963-c996ace4428b","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"8e1788eb","trusted":true},"outputs":[],"source":["# helper = pd.read_csv(\"/kaggle/input/mlprojectdataset/test_tfidf_features.csv\")\n","# helper"]},{"cell_type":"code","execution_count":138,"metadata":{"cell_id":"989a7923bb7d40c095cc9486a8a4cb85","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"4af41c9f","tags":[],"trusted":true},"outputs":[],"source":["# helper = pd.read_csv(\"/work/50007-2022/test_tfidf_features.csv\")\n","# helper"]},{"cell_type":"code","execution_count":139,"metadata":{"cell_id":"00064-7b610f6e-502e-45a7-a991-6cf42793a07a","deepnote_cell_height":171,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"ac8ff272","trusted":true},"outputs":[],"source":["# x_tester = helper.iloc[:,1:]\n","# pca2 = PCA(n_components=1000)\n","# pca2.fit(x_tester)\n","# x_trans = pca2.transform(x_tester)\n","# results = predict(x_trans, w, b)\n","# results"]},{"cell_type":"code","execution_count":140,"metadata":{"cell_id":"00065-8067a7df-448b-42ed-b51f-ce4778f4608a","deepnote_cell_height":171,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"1f0e93c9","trusted":true},"outputs":[],"source":["# idsindf = helper.iloc[:, 0].to_frame()\n","# resultsindf = pd.DataFrame(results)\n","# print(resultsindf.shape, idsindf.shape)\n","# df_results = pd.concat([idsindf,resultsindf], axis =1)\n","# df_results = df_results.rename(columns={0: 'label'})\n","# print(df_results)"]},{"cell_type":"code","execution_count":141,"metadata":{"cell_id":"00066-fb033115-8594-43b1-a172-40693a856d9b","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"source_hash":"50349075","trusted":true},"outputs":[],"source":["# # Write to csv\n","# df_results.to_csv(\"LogRed_Prediction3.csv\", index=False)"]},{"cell_type":"markdown","metadata":{},"source":["# Task 3"]},{"cell_type":"markdown","metadata":{},"source":["## LightGBM Decision Tree (Gradient Boosting Algorithm)"]},{"cell_type":"code","execution_count":69,"metadata":{},"outputs":[{"name":"stderr","output_type":"stream","text":["Intel(R) Extension for Scikit-learn* enabled (https://github.com/intel/scikit-learn-intelex)\n"]}],"source":["from sklearnex import patch_sklearn, unpatch_sklearn\n","patch_sklearn()\n","import numpy as np\n","import pandas as pd\n","\n","from imblearn.metrics import classification_report_imbalanced\n","from imblearn.pipeline import make_pipeline\n","from sklearn.model_selection import train_test_split, cross_val_score, RepeatedStratifiedKFold\n","from sklearn.metrics import f1_score\n","from sklearn.feature_selection import chi2, SelectPercentile\n","from sklearn.preprocessing import RobustScaler\n","\n","from lightgbm import LGBMClassifier as LGBM\n","from catboost import CatBoostClassifier as CatBoost\n","from xgboost import (XGBClassifier as XGB, XGBRFClassifier as RandomForest)\n","from sklearn.decomposition import TruncatedSVD\n","from sklearn.tree import DecisionTreeClassifier as DecisionTree\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.ensemble import (AdaBoostClassifier as AdaBoost, HistGradientBoostingClassifier as HistGradBoost, StackingClassifier, VotingClassifier) # GradientBoostingClassifier is too slow for large datasets"]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":["def score(y, y_hat):\n","    accuracy = np.sum(y == y_hat) / np.shape(y)[0]\n","    f1score = f1_score(y, y_hat, average='macro')\n","\n","    print(f\"Accuracy: {accuracy}\")\n","    print(f\"Macro-F1 score: {f1score}\")\n","\n","    # Return Macro-F1 score of the model\n","    return f1score"]},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>17180</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>17181</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>17182</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>17183</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>17184</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5002 columns</p>\n","</div>"],"text/plain":["          id  label    0    1    2    3    4    5    6    7  ...  4990  4991  \\\n","0          1      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1          2      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2          3      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3          4      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4          5      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...      ...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  17180      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  17181      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  17182      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  17183      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  17184      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5002 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 18.3 s\n","Wall time: 18.3 s\n"]}],"source":["%%time\n","df_train = pd.read_csv(r\"./source/train_tfidf_features.csv\")\n","display(df_train)"]},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["0    10633\n","1     6551\n","Name: label, dtype: int64\n"]}],"source":["count_label = df_train['label'].value_counts(dropna=False) # Unique labels\n","print(count_label)"]},{"cell_type":"code","execution_count":6,"metadata":{},"outputs":[{"data":{"text/plain":["array([[0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       ...,\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.]])"]},"metadata":{},"output_type":"display_data"}],"source":["X = df_train.iloc[:, 2:].to_numpy()\n","display(X)"]},{"cell_type":"code","execution_count":7,"metadata":{},"outputs":[{"data":{"text/plain":["array([1, 0, 1, ..., 1, 1, 0], dtype=int64)"]},"metadata":{},"output_type":"display_data"}],"source":["y = df_train.iloc[:, 1].to_numpy()\n","display(y)"]},{"cell_type":"code","execution_count":96,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 2250), y_train: (12888,)\n","X_test: (4296, 2250), y_test: (4296,)\n","CPU times: total: 3 s\n","Wall time: 1.98 s\n"]}],"source":["%%time\n","select = SelectPercentile(chi2, percentile=45).fit(X, y)\n","X_new = select.transform(X)\n","X_train, X_valid, y_train, y_valid = train_test_split(X_new, y, test_size=0.25, random_state=100)\n","scaler = RobustScaler(with_centering=False, unit_variance=True).fit(X_train)\n","X_train = scaler.transform(X_train)\n","X_valid = scaler.transform(X_valid)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_valid.shape}, y_test: {y_valid.shape}\")"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[],"source":["# print(results)\n","# max(results, key=results.get)"]},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>2240</th>\n","      <th>2241</th>\n","      <th>2242</th>\n","      <th>2243</th>\n","      <th>2244</th>\n","      <th>2245</th>\n","      <th>2246</th>\n","      <th>2247</th>\n","      <th>2248</th>\n","      <th>2249</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>...</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>0.001299</td>\n","      <td>0.000611</td>\n","      <td>0.000625</td>\n","      <td>0.000200</td>\n","      <td>0.000250</td>\n","      <td>0.000291</td>\n","      <td>0.000288</td>\n","      <td>0.000123</td>\n","      <td>0.000274</td>\n","      <td>0.000318</td>\n","      <td>...</td>\n","      <td>0.000783</td>\n","      <td>0.000257</td>\n","      <td>0.000688</td>\n","      <td>0.000815</td>\n","      <td>0.000222</td>\n","      <td>0.000311</td>\n","      <td>0.000096</td>\n","      <td>0.000309</td>\n","      <td>0.001044</td>\n","      <td>0.000367</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>0.025179</td>\n","      <td>0.017126</td>\n","      <td>0.019198</td>\n","      <td>0.010162</td>\n","      <td>0.011647</td>\n","      <td>0.015865</td>\n","      <td>0.014736</td>\n","      <td>0.008047</td>\n","      <td>0.014058</td>\n","      <td>0.013741</td>\n","      <td>...</td>\n","      <td>0.022520</td>\n","      <td>0.013331</td>\n","      <td>0.018665</td>\n","      <td>0.020641</td>\n","      <td>0.011547</td>\n","      <td>0.014753</td>\n","      <td>0.008023</td>\n","      <td>0.013293</td>\n","      <td>0.025268</td>\n","      <td>0.014015</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>0.756548</td>\n","      <td>0.724059</td>\n","      <td>0.742259</td>\n","      <td>0.569983</td>\n","      <td>0.643714</td>\n","      <td>1.319443</td>\n","      <td>0.847275</td>\n","      <td>0.551220</td>\n","      <td>0.813805</td>\n","      <td>0.689282</td>\n","      <td>...</td>\n","      <td>1.043452</td>\n","      <td>0.880900</td>\n","      <td>0.744073</td>\n","      <td>0.717912</td>\n","      <td>0.769711</td>\n","      <td>0.824391</td>\n","      <td>0.798826</td>\n","      <td>0.632732</td>\n","      <td>1.147449</td>\n","      <td>0.654102</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 2250 columns</p>\n","</div>"],"text/plain":["               0             1             2             3             4     \\\n","count  12888.000000  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean       0.001299      0.000611      0.000625      0.000200      0.000250   \n","std        0.025179      0.017126      0.019198      0.010162      0.011647   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.756548      0.724059      0.742259      0.569983      0.643714   \n","\n","               5             6             7             8             9     \\\n","count  12888.000000  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean       0.000291      0.000288      0.000123      0.000274      0.000318   \n","std        0.015865      0.014736      0.008047      0.014058      0.013741   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        1.319443      0.847275      0.551220      0.813805      0.689282   \n","\n","       ...          2240          2241          2242          2243  \\\n","count  ...  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean   ...      0.000783      0.000257      0.000688      0.000815   \n","std    ...      0.022520      0.013331      0.018665      0.020641   \n","min    ...      0.000000      0.000000      0.000000      0.000000   \n","25%    ...      0.000000      0.000000      0.000000      0.000000   \n","50%    ...      0.000000      0.000000      0.000000      0.000000   \n","75%    ...      0.000000      0.000000      0.000000      0.000000   \n","max    ...      1.043452      0.880900      0.744073      0.717912   \n","\n","               2244          2245          2246          2247          2248  \\\n","count  12888.000000  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean       0.000222      0.000311      0.000096      0.000309      0.001044   \n","std        0.011547      0.014753      0.008023      0.013293      0.025268   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.769711      0.824391      0.798826      0.632732      1.147449   \n","\n","               2249  \n","count  12888.000000  \n","mean       0.000367  \n","std        0.014015  \n","min        0.000000  \n","25%        0.000000  \n","50%        0.000000  \n","75%        0.000000  \n","max        0.654102  \n","\n","[8 rows x 2250 columns]"]},"metadata":{},"output_type":"display_data"},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>2240</th>\n","      <th>2241</th>\n","      <th>2242</th>\n","      <th>2243</th>\n","      <th>2244</th>\n","      <th>2245</th>\n","      <th>2246</th>\n","      <th>2247</th>\n","      <th>2248</th>\n","      <th>2249</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.0</td>\n","      <td>4296.0</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>...</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>0.001853</td>\n","      <td>0.000774</td>\n","      <td>0.000383</td>\n","      <td>0.000259</td>\n","      <td>0.000484</td>\n","      <td>0.000124</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000131</td>\n","      <td>0.000143</td>\n","      <td>...</td>\n","      <td>0.000771</td>\n","      <td>0.000143</td>\n","      <td>0.000734</td>\n","      <td>0.000636</td>\n","      <td>0.000091</td>\n","      <td>0.000158</td>\n","      <td>0.000116</td>\n","      <td>0.000473</td>\n","      <td>0.000695</td>\n","      <td>0.000285</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>0.029581</td>\n","      <td>0.019245</td>\n","      <td>0.012835</td>\n","      <td>0.012035</td>\n","      <td>0.019628</td>\n","      <td>0.008124</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.008593</td>\n","      <td>0.009360</td>\n","      <td>...</td>\n","      <td>0.021040</td>\n","      <td>0.009395</td>\n","      <td>0.020028</td>\n","      <td>0.021682</td>\n","      <td>0.005973</td>\n","      <td>0.010326</td>\n","      <td>0.007576</td>\n","      <td>0.018140</td>\n","      <td>0.018707</td>\n","      <td>0.013224</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>0.738732</td>\n","      <td>0.563316</td>\n","      <td>0.550708</td>\n","      <td>0.581338</td>\n","      <td>1.071790</td>\n","      <td>0.532496</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.563232</td>\n","      <td>0.613522</td>\n","      <td>...</td>\n","      <td>0.756139</td>\n","      <td>0.615815</td>\n","      <td>0.686754</td>\n","      <td>1.183895</td>\n","      <td>0.391466</td>\n","      <td>0.676810</td>\n","      <td>0.496562</td>\n","      <td>0.832780</td>\n","      <td>0.594288</td>\n","      <td>0.654131</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 2250 columns</p>\n","</div>"],"text/plain":["              0            1            2            3            4     \\\n","count  4296.000000  4296.000000  4296.000000  4296.000000  4296.000000   \n","mean      0.001853     0.000774     0.000383     0.000259     0.000484   \n","std       0.029581     0.019245     0.012835     0.012035     0.019628   \n","min       0.000000     0.000000     0.000000     0.000000     0.000000   \n","25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","max       0.738732     0.563316     0.550708     0.581338     1.071790   \n","\n","              5       6       7            8            9     ...  \\\n","count  4296.000000  4296.0  4296.0  4296.000000  4296.000000  ...   \n","mean      0.000124     0.0     0.0     0.000131     0.000143  ...   \n","std       0.008124     0.0     0.0     0.008593     0.009360  ...   \n","min       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","25%       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","50%       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","75%       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","max       0.532496     0.0     0.0     0.563232     0.613522  ...   \n","\n","              2240         2241         2242         2243         2244  \\\n","count  4296.000000  4296.000000  4296.000000  4296.000000  4296.000000   \n","mean      0.000771     0.000143     0.000734     0.000636     0.000091   \n","std       0.021040     0.009395     0.020028     0.021682     0.005973   \n","min       0.000000     0.000000     0.000000     0.000000     0.000000   \n","25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","max       0.756139     0.615815     0.686754     1.183895     0.391466   \n","\n","              2245         2246         2247         2248         2249  \n","count  4296.000000  4296.000000  4296.000000  4296.000000  4296.000000  \n","mean      0.000158     0.000116     0.000473     0.000695     0.000285  \n","std       0.010326     0.007576     0.018140     0.018707     0.013224  \n","min       0.000000     0.000000     0.000000     0.000000     0.000000  \n","25%       0.000000     0.000000     0.000000     0.000000     0.000000  \n","50%       0.000000     0.000000     0.000000     0.000000     0.000000  \n","75%       0.000000     0.000000     0.000000     0.000000     0.000000  \n","max       0.676810     0.496562     0.832780     0.594288     0.654131  \n","\n","[8 rows x 2250 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["display(pd.DataFrame(X_train).describe())\n","display(pd.DataFrame(X_valid).describe())"]},{"cell_type":"markdown","metadata":{},"source":[]},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["0.9750809144340586\n","CPU times: total: 3min 17s\n","Wall time: 28.3 s\n"]}],"source":["%%time\n","svd = TruncatedSVD(n_components=1500, random_state=100)\n","svd.fit(X_train)\n","X_red = svd.transform(X_train)\n","print(np.sum(svd.explained_variance_ratio_))"]},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[5.51782125e-01 5.46996746e-03 4.86072801e-03 ... 5.24781104e-05\n"," 5.22818921e-05 5.21341159e-05]\n"]}],"source":["print(svd.explained_variance_ratio_)"]},{"cell_type":"code","execution_count":14,"metadata":{},"outputs":[],"source":["lgbm = LGBM(boosting_type='gbdt', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight='balanced', reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=5, random_state=100)"]},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.698 (0.012)\n","CPU times: total: 453 ms\n","Wall time: 6min 23s\n"]}],"source":["%%time\n","n_scores = cross_val_score(lgbm, X_red, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":16,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7216014897579144\n","Macro-F1 score: 0.7017628096479489\n","CPU times: total: 2min 7s\n","Wall time: 13.9 s\n"]},{"data":{"text/plain":["0.7017628096479489"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","lgbm.fit(X_red, y_train)\n","X_valid_red = svd.transform(X_valid)\n","score(y_valid, lgbm.predict(X_valid_red))"]},{"cell_type":"markdown","metadata":{},"source":[]},{"cell_type":"code","execution_count":17,"metadata":{},"outputs":[],"source":["lgbm = LGBM(boosting_type='gbdt', learning_rate=0.13, num_leaves=162, max_depth=193, colsample_bytree=0.6, subsample=0.6, min_child_samples=3, min_child_weight=0.02, \\\n","    min_split_gain=0.02, max_delta_step=0.6, n_estimators=115, scale_pos_weight=1.48, reg_lambda=0, random_state=100, reg_alpha=1.3, n_jobs=-1)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=5, random_state=100)\n","# boosting_type='gbdt', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight=None, reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1"]},{"cell_type":"code","execution_count":18,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.706 (0.010)\n","CPU times: total: 344 ms\n","Wall time: 59.7 s\n"]}],"source":["%%time\n","n_scores = cross_val_score(lgbm, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":19,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7295158286778398\n","Macro-F1 score: 0.7147951441578149\n","CPU times: total: 40 s\n","Wall time: 4.53 s\n"]},{"data":{"text/plain":["0.7147951441578149"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","lgbm.fit(X_train, y_train)\n","score(y_valid, lgbm.predict(X_valid))\n","# 0.7084803795711445 num_leaves 200\n","# 0.7090648077930747 num_leaves 205"]},{"cell_type":"code","execution_count":20,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.76      0.67      0.78      0.72      0.52      2687\n","          1       0.63      0.67      0.76      0.65      0.72      0.51      1609\n","\n","avg / total       0.73      0.73      0.71      0.73      0.72      0.51      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, lgbm.predict(X_valid)))"]},{"cell_type":"code","execution_count":21,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.76      0.67      0.78      0.72      0.52      2687\n","          1       0.63      0.67      0.76      0.65      0.72      0.51      1609\n","\n","avg / total       0.73      0.73      0.71      0.73      0.72      0.51      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, lgbm.predict(X_valid)))"]},{"cell_type":"code","execution_count":22,"metadata":{},"outputs":[],"source":["estimators = []\n","estimators.append(('gbdt', LGBM(boosting_type='gbdt', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight='balanced', reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)))\n","estimators.append(('dart', LGBM(boosting_type='dart', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight='balanced', reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)))\n","estimators.append(('goss', LGBM(boosting_type='goss', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight='balanced', reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)))"]},{"cell_type":"code","execution_count":23,"metadata":{},"outputs":[],"source":["final = LogisticRegression(random_state = 100, class_weight='balanced')\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)\n","stack = StackingClassifier(estimators=estimators, final_estimator=final, cv=10, stack_method=\"auto\", n_jobs=-1, passthrough=False, verbose=0)"]},{"cell_type":"code","execution_count":24,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7125232774674115\n","Macro-F1 score: 0.7016259808256581\n","CPU times: total: 6.44 s\n","Wall time: 18.5 s\n"]},{"data":{"text/plain":["0.7016259808256581"]},"execution_count":24,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","stack.fit(X_train, y_train)\n","score(y_valid, stack.predict(X_valid))"]},{"cell_type":"code","execution_count":25,"metadata":{},"outputs":[],"source":["cboost = CatBoost(boosting_type=\"Plain\", loss_function=\"Logloss\", max_depth=16, learning_rate=0.2, grow_policy='Depthwise', colsample_bylevel=0.39,\\\n","     n_estimators=100, min_child_samples=58, scale_pos_weight=1.52, reg_lambda=1.33, bootstrap_type='MVS', mvs_reg=9.75, random_state=100, verbose=0)\n","# loss_function=\"Logloss\", max_depth=None, use_best_model=True, task_type=\"GPU\", devices='1', set max_depth to 4 due to memory error\n","#from catboost.utils import get_gpu_device_count\n","# print('I see %i GPU devices' % get_gpu_device_count())\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=5, random_state=100)"]},{"cell_type":"code","execution_count":26,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.708 (0.012)\n","CPU times: total: 359 ms\n","Wall time: 4min 19s\n"]}],"source":["%%time\n","n_scores = cross_val_score(cboost, X_train, y_train, scoring=\"f1_macro\", cv=cv, n_jobs=-1, error_score=\"raise\")\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":27,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7257914338919925\n","Macro-F1 score: 0.7073652739070306\n","CPU times: total: 57.6 s\n","Wall time: 8.57 s\n"]},{"data":{"text/plain":["0.7073652739070306"]},"execution_count":27,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","cboost.fit(X_train, y_train)\n","score(y_valid, cboost.predict(X_valid))\n","# 0.6864229482217066"]},{"cell_type":"code","execution_count":28,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.78      0.78      0.63      0.78      0.70      0.50      2687\n","          1       0.63      0.63      0.78      0.63      0.70      0.49      1609\n","\n","avg / total       0.73      0.73      0.69      0.73      0.70      0.50      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, cboost.predict(X_valid)))"]},{"cell_type":"code","execution_count":56,"metadata":{},"outputs":[],"source":["xgboost = XGB(learning_rate=0.25, booster=\"gbtree\", gamma=0.6, min_child_weight=0.02, max_delta_step=0.6, \\\n","    colsample_bylevel=0.35, max_leaves=120, eval_metric=\"error\", tree_method=\"hist\", max_depth=12, subsample=0.65, \\\n","    scale_pos_weight=1.5,  grow_policy=\"depthwise\", n_estimators=100, reg_alpha=0, reg_lambda=0, random_state=100, verbosity=1, n_jobs=-1)\n","# num_parallel_tree=0, check GPU supported algorithms (tree_method=\"gpu_hist\", gpu_id=1), max_leaves=40, max_depth=31, \n","# gamma, min_child_weight, max_delta_step, subsample, colsample_bylevel, process_type=update, max_leaves, eval_metric=error\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":57,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.694 (0.014)\n","CPU times: total: 406 ms\n","Wall time: 1min 27s\n"]}],"source":["%%time\n","n_scores = cross_val_score(xgboost, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":58,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7143854748603352\n","Macro-F1 score: 0.6957578553920669\n","CPU times: total: 1min 14s\n","Wall time: 9.38 s\n"]},{"data":{"text/plain":["0.6957578553920669"]},"execution_count":58,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","xgboost.fit(X_train, y_train)\n","score(y_valid, xgboost.predict(X_valid))"]},{"cell_type":"code","execution_count":59,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.77      0.77      0.62      0.77      0.69      0.49      2687\n","          1       0.62      0.62      0.77      0.62      0.69      0.47      1609\n","\n","avg / total       0.71      0.71      0.68      0.71      0.69      0.48      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, xgboost.predict(X_valid)))"]},{"cell_type":"code","execution_count":33,"metadata":{},"outputs":[],"source":["dtrees = DecisionTree(criterion=\"gini\", splitter=\"best\", max_features=\"sqrt\", max_depth=20, class_weight=\"balanced\", max_leaf_nodes=40, random_state=100)\n","# criterion = \"log_loss\" or \"entropy\", min_samples_split, min_samples_leaf, ccp_alpha\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":34,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.556 (0.030)\n","CPU times: total: 359 ms\n","Wall time: 6.88 s\n"]}],"source":["%%time\n","n_scores = cross_val_score(dtrees, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":35,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6026536312849162\n","Macro-F1 score: 0.591656606022154\n","CPU times: total: 266 ms\n","Wall time: 333 ms\n"]},{"data":{"text/plain":["0.591656606022154"]},"execution_count":35,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","dtrees.fit(X_train, y_train)\n","score(y_valid, dtrees.predict(X_valid))"]},{"cell_type":"code","execution_count":79,"metadata":{},"outputs":[],"source":["randforest = RandomForest(learning_rate=0.25, booster=\"gbtree\", gamma=0.5, min_child_weight=0.02, max_delta_step=0.7, colsample_bylevel=0.65, max_leaves=120, eval_metric=\"error\", tree_method=\"hist\", max_depth=20, subsample=0.65, \\\n","    scale_pos_weight=1.63,  grow_policy=\"depthwise\", n_estimators=100, reg_alpha=0, reg_lambda=0, random_state=100, verbosity=1, n_jobs=-1, num_parallel_tree=2)\n","# colsample_bytree, colsample_bynode, \n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":80,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.645 (0.014)\n","CPU times: total: 359 ms\n","Wall time: 2min 34s\n"]}],"source":["%%time\n","n_scores = cross_val_score(randforest, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":81,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6606145251396648\n","Macro-F1 score: 0.6467869493438152\n","CPU times: total: 2min 22s\n","Wall time: 22.7 s\n"]},{"data":{"text/plain":["0.6467869493438152"]},"execution_count":81,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","randforest.fit(X_train, y_train)\n","score(y_valid, randforest.predict(X_valid))"]},{"cell_type":"code","execution_count":78,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.73      0.77      0.53      0.75      0.64      0.42      2687\n","          1       0.58      0.53      0.77      0.55      0.64      0.40      1609\n","\n","avg / total       0.67      0.68      0.62      0.68      0.64      0.41      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, randforest.predict(X_valid)))"]},{"cell_type":"code","execution_count":40,"metadata":{},"outputs":[],"source":["base = DecisionTree(criterion=\"gini\", splitter=\"best\", max_features=\"sqrt\", max_depth=35, class_weight=\"balanced\", max_leaf_nodes=40, random_state=100)\n","adaboost = AdaBoost(base_estimator=base, n_estimators=100, learning_rate=0.5, random_state=100)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":41,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.685 (0.011)\n","CPU times: total: 328 ms\n","Wall time: 6min\n"]}],"source":["%%time\n","n_scores = cross_val_score(adaboost, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":42,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7099627560521415\n","Macro-F1 score: 0.6924188671569287\n","CPU times: total: 23.9 s\n","Wall time: 24 s\n"]},{"data":{"text/plain":["0.6924188671569287"]},"execution_count":42,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","adaboost.fit(X_train, y_train)\n","score(y_valid, adaboost.predict(X_valid))"]},{"cell_type":"code","execution_count":43,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.77      0.76      0.63      0.77      0.69      0.48      2687\n","          1       0.61      0.63      0.76      0.62      0.69      0.47      1609\n","\n","avg / total       0.71      0.71      0.68      0.71      0.69      0.48      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, adaboost.predict(X_valid)))"]},{"cell_type":"code","execution_count":44,"metadata":{},"outputs":[],"source":["histboost = HistGradBoost(loss=\"log_loss\", learning_rate=0.3, max_depth=40, max_leaf_nodes=50, warm_start=True, scoring=\"f1_macro\", verbose=100, random_state=100)\n","# max_iter, max_depth, min_samples_leaf (default = 20), l2_regularization, max_bins, monotonic_cst, validation_fraction, n_iter_no_change, tol\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":45,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.676 (0.012)\n","CPU times: total: 234 ms\n","Wall time: 4min 21s\n"]}],"source":["%%time\n","n_scores = cross_val_score(histboost, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":46,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Binning 0.209 GB of training data: 2.392 s\n","Binning 0.023 GB of validation data: 0.152 s\n","Fitting gradient boosted rounds:\n","[1/100] 1 tree, 50 leaves, max depth = 34, train score: 0.43445, val score: 0.44815, in 1.215s\n","[2/100] 1 tree, 50 leaves, max depth = 38, train score: 0.61203, val score: 0.62228, in 1.352s\n","[3/100] 1 tree, 50 leaves, max depth = 37, train score: 0.64145, val score: 0.64856, in 1.360s\n","[4/100] 1 tree, 50 leaves, max depth = 40, train score: 0.66323, val score: 0.66145, in 1.326s\n","[5/100] 1 tree, 50 leaves, max depth = 40, train score: 0.67868, val score: 0.67172, in 1.293s\n","[6/100] 1 tree, 50 leaves, max depth = 40, train score: 0.69076, val score: 0.68512, in 1.234s\n","[7/100] 1 tree, 50 leaves, max depth = 40, train score: 0.69838, val score: 0.68181, in 1.265s\n","[8/100] 1 tree, 50 leaves, max depth = 40, train score: 0.70584, val score: 0.68618, in 1.324s\n","[9/100] 1 tree, 50 leaves, max depth = 40, train score: 0.71140, val score: 0.68411, in 1.315s\n","[10/100] 1 tree, 50 leaves, max depth = 40, train score: 0.72053, val score: 0.68369, in 1.344s\n","[11/100] 1 tree, 50 leaves, max depth = 40, train score: 0.72660, val score: 0.68349, in 1.320s\n","[12/100] 1 tree, 50 leaves, max depth = 40, train score: 0.73385, val score: 0.68698, in 1.285s\n","[13/100] 1 tree, 50 leaves, max depth = 40, train score: 0.73721, val score: 0.68772, in 1.257s\n","[14/100] 1 tree, 50 leaves, max depth = 40, train score: 0.74048, val score: 0.68531, in 1.308s\n","[15/100] 1 tree, 50 leaves, max depth = 40, train score: 0.74246, val score: 0.68634, in 1.365s\n","[16/100] 1 tree, 50 leaves, max depth = 40, train score: 0.74611, val score: 0.68979, in 1.389s\n","[17/100] 1 tree, 50 leaves, max depth = 40, train score: 0.75211, val score: 0.68769, in 1.356s\n","[18/100] 1 tree, 50 leaves, max depth = 40, train score: 0.75550, val score: 0.68630, in 1.339s\n","[19/100] 1 tree, 50 leaves, max depth = 40, train score: 0.75681, val score: 0.68936, in 1.307s\n","[20/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76066, val score: 0.69513, in 1.286s\n","[21/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76221, val score: 0.69646, in 1.239s\n","[22/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76359, val score: 0.69847, in 1.417s\n","[23/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76594, val score: 0.69868, in 1.315s\n","[24/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76945, val score: 0.69898, in 1.316s\n","[25/100] 1 tree, 50 leaves, max depth = 36, train score: 0.77229, val score: 0.69968, in 1.306s\n","[26/100] 1 tree, 50 leaves, max depth = 40, train score: 0.77534, val score: 0.69758, in 1.334s\n","[27/100] 1 tree, 50 leaves, max depth = 40, train score: 0.77894, val score: 0.69659, in 1.375s\n","[28/100] 1 tree, 50 leaves, max depth = 40, train score: 0.78099, val score: 0.69728, in 1.325s\n","[29/100] 1 tree, 50 leaves, max depth = 40, train score: 0.78181, val score: 0.69629, in 1.310s\n","[30/100] 1 tree, 50 leaves, max depth = 33, train score: 0.78787, val score: 0.69758, in 1.424s\n","[31/100] 1 tree, 50 leaves, max depth = 40, train score: 0.78937, val score: 0.69986, in 1.343s\n","[32/100] 1 tree, 50 leaves, max depth = 40, train score: 0.79064, val score: 0.69589, in 1.339s\n","[33/100] 1 tree, 50 leaves, max depth = 40, train score: 0.79153, val score: 0.69449, in 1.334s\n","[34/100] 1 tree, 50 leaves, max depth = 40, train score: 0.79236, val score: 0.69180, in 1.371s\n","[35/100] 1 tree, 50 leaves, max depth = 40, train score: 0.79246, val score: 0.69171, in 1.332s\n","[36/100] 1 tree, 50 leaves, max depth = 37, train score: 0.79286, val score: 0.69240, in 1.504s\n","[37/100] 1 tree, 50 leaves, max depth = 40, train score: 0.79366, val score: 0.68992, in 1.425s\n","[38/100] 1 tree, 50 leaves, max depth = 34, train score: 0.79407, val score: 0.68980, in 1.440s\n","[39/100] 1 tree, 50 leaves, max depth = 40, train score: 0.79500, val score: 0.69009, in 1.418s\n","[40/100] 1 tree, 50 leaves, max depth = 40, train score: 0.79690, val score: 0.69177, in 1.579s\n","[41/100] 1 tree, 50 leaves, max depth = 33, train score: 0.80014, val score: 0.69345, in 1.357s\n","Fit 41 trees in 58.784 s, (2050 total leaves)\n","Time spent computing histograms: 42.457s\n","Time spent finding best splits:  0.908s\n","Time spent applying splits:      0.632s\n","Time spent predicting:           0.008s\n","Accuracy: 0.715782122905028\n","Macro-F1 score: 0.6847935369111147\n","CPU times: total: 4min 53s\n","Wall time: 58.9 s\n"]},{"data":{"text/plain":["0.6847935369111147"]},"execution_count":46,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","histboost.fit(X_train, y_train)\n","score(y_valid, histboost.predict(X_valid))"]},{"cell_type":"code","execution_count":47,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.75      0.82      0.54      0.78      0.66      0.45      2687\n","          1       0.64      0.54      0.82      0.59      0.66      0.43      1609\n","\n","avg / total       0.71      0.72      0.64      0.71      0.66      0.45      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, histboost.predict(X_valid)))"]},{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[],"source":["from sklearn.ensemble import VotingClassifier, StackingClassifier\n","\n","from sklearn.naive_bayes import MultinomialNB, ComplementNB # Other NB have poor performance\n","from sklearn.pipeline import make_pipeline\n","from imblearn.over_sampling import SMOTE, ADASYN\n","from imblearn.metrics import classification_report_imbalanced\n","from imblearn.pipeline import make_pipeline as make_pipeline_imb\n","\n","from sklearn.kernel_approximation import Nystroem\n","from sklearn.svm import SVC, NuSVC, LinearSVC # One-Class SVM too slow\n","from sklearn.neighbors import KNeighborsClassifier as KNN\n","from sklearn.linear_model import LogisticRegression, LogisticRegressionCV"]},{"cell_type":"code","execution_count":110,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 5001 columns</p>\n","</div>"],"text/plain":["         id    0    1    2    3    4    5    6    7    8  ...  4990  4991  \\\n","0     17185  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1     17186  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2     17187  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3     17188  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4     17189  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...     ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","4291  21476  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4292  21477  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4293  21478  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4294  21479  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4295  21480  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","      4992  4993  4994  4995  4996  4997  4998  4999  \n","0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...    ...   ...   ...   ...   ...   ...   ...   ...  \n","4291   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4292   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4293   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4294   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4295   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[4296 rows x 5001 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["df_test = pd.read_csv(r\"./source/test_tfidf_features.csv\")\n","display(df_test)"]},{"cell_type":"code","execution_count":111,"metadata":{},"outputs":[{"data":{"text/plain":["array([[0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       ...,\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.]])"]},"metadata":{},"output_type":"display_data"}],"source":["predict_label = df_test.iloc[:, 1:].to_numpy()\n","display(predict_label)"]},{"cell_type":"code","execution_count":82,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.736731843575419\n","Macro-F1 score: 0.7176431536465723\n","CPU times: total: 9.44 s\n","Wall time: 1min 38s\n"]},{"data":{"text/plain":["0.7176431536465723"]},"execution_count":82,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","models = [('lgbm', lgbm), ('catboost', cboost), ('xgboost', xgboost), ('randforest', randforest), ('adaboost', adaboost), ('histboost', histboost)] \n","votes = VotingClassifier(estimators=models, voting='soft', n_jobs=-1, verbose=True)\n","votes = votes.fit(X_train, y_train)\n","score(y_valid, votes.predict(X_valid))"]},{"cell_type":"code","execution_count":83,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.80      0.64      0.79      0.71      0.52      2687\n","          1       0.65      0.64      0.80      0.64      0.71      0.50      1609\n","\n","avg / total       0.74      0.74      0.70      0.74      0.71      0.51      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, votes.predict(X_valid)))"]},{"cell_type":"code","execution_count":84,"metadata":{},"outputs":[],"source":["final = LogisticRegression(random_state = 100)\n","stacktree = StackingClassifier(estimators=models, final_estimator=final, cv=\"prefit\", stack_method=\"auto\", n_jobs=-1, passthrough=False, verbose=0)"]},{"cell_type":"code","execution_count":85,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7332402234636871\n","Macro-F1 score: 0.7128030660996483\n","CPU times: total: 29.5 s\n","Wall time: 12.4 s\n"]},{"data":{"text/plain":["0.7128030660996483"]},"execution_count":85,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","stacktree.fit(X_train, y_train)\n","score(y_valid, stacktree.predict(X_valid))"]},{"cell_type":"code","execution_count":86,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.78      0.80      0.62      0.79      0.71      0.51      2687\n","          1       0.65      0.62      0.80      0.64      0.71      0.49      1609\n","\n","avg / total       0.73      0.73      0.69      0.73      0.71      0.50      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, stacktree.predict(X_valid)))"]},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[{"ename":"NameError","evalue":"name 'votes' is not defined","output_type":"error","traceback":["\u001b[1;31m---------------------------------------------------------------------------\u001b[0m","\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)","File \u001b[1;32m<timed exec>:1\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n","\u001b[1;31mNameError\u001b[0m: name 'votes' is not defined"]}],"source":["%%time\n","results = votes.predict(predict_label)\n","display(results)"]},{"cell_type":"code","execution_count":79,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["(4296, 1) (4296, 1)\n","\n","\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 2 columns</p>\n","</div>"],"text/plain":["         id  label\n","0     17185      1\n","1     17186      0\n","2     17187      1\n","3     17188      0\n","4     17189      0\n","...     ...    ...\n","4291  21476      1\n","4292  21477      1\n","4293  21478      1\n","4294  21479      0\n","4295  21480      0\n","\n","[4296 rows x 2 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 15.6 ms\n","Wall time: 10 ms\n"]},{"data":{"text/plain":["0    2750\n","1    1546\n","Name: label, dtype: int64"]},"execution_count":79,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","df_ids = df_test.iloc[:, 0].to_frame()\n","df_results = pd.DataFrame(results)\n","print(df_results.shape, df_ids.shape)\n","print(\"\\n\")\n","    \n","df_submission = pd.concat([df_ids, df_results], axis =1)\n","df_submission = df_submission.rename(columns={0: 'label'})\n","\n","display(df_submission)\n","df_submission['label'].value_counts()"]},{"cell_type":"code","execution_count":80,"metadata":{},"outputs":[],"source":["# df_submission.to_csv(f\"Voting_Ensemble_Predictions.csv\", index=False)"]},{"cell_type":"markdown","metadata":{},"source":[]},{"cell_type":"code","execution_count":98,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.715316573556797\n","Macro-F1 score: 0.7117328238097133\n","CPU times: total: 10.5 s\n","Wall time: 2.23 s\n"]},{"data":{"text/plain":["0.7117328238097133"]},"execution_count":98,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bayes = make_pipeline_imb(SMOTE(sampling_strategy='minority', k_neighbors=4, random_state=100, n_jobs=-1), MultinomialNB(alpha=13)) # RandomUnderSampler(random_state=100), \n","bayes.fit(X_train, y_train)\n","score(y_valid, bayes.predict(X_valid))"]},{"cell_type":"code","execution_count":151,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.85      0.66      0.81      0.74      0.73      0.53      2687\n","          1       0.59      0.81      0.66      0.68      0.73      0.54      1609\n","\n","avg / total       0.75      0.72      0.75      0.72      0.73      0.53      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, bayes.predict(X_valid)))"]},{"cell_type":"code","execution_count":99,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.707169459962756\n","Macro-F1 score: 0.7051216939485097\n","CPU times: total: 34.9 s\n","Wall time: 5.12 s\n"]},{"data":{"text/plain":["0.7051216939485097"]},"execution_count":99,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","cbayes = make_pipeline_imb(ADASYN(sampling_strategy='minority', n_neighbors=6, random_state=100, n_jobs=-1), ComplementNB(alpha=10)) #\n","cbayes.fit(X_train, y_train)\n","score(y_valid, cbayes.predict(X_valid))"]},{"cell_type":"code","execution_count":152,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.86      0.63      0.83      0.73      0.73      0.52      2687\n","          1       0.58      0.83      0.63      0.68      0.73      0.54      1609\n","\n","avg / total       0.76      0.71      0.76      0.71      0.73      0.52      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, cbayes.predict(X_valid)))"]},{"cell_type":"code","execution_count":100,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7332402234636871\n","Macro-F1 score: 0.7251284854892297\n","CPU times: total: 12.6 s\n","Wall time: 2.38 s\n"]},{"data":{"text/plain":["0.7251284854892297"]},"execution_count":100,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","lsvm = make_pipeline_imb(SMOTE(sampling_strategy='minority', k_neighbors=8, random_state=100), LinearSVC(penalty=\"l2\", loss=\"squared_hinge\", dual=False, C=1.0, class_weight=\"balanced\", verbose=0, random_state=100, max_iter=1000))\n","lsvm.fit(X_train, y_train)\n","score(y_valid, lsvm.predict(X_valid))"]},{"cell_type":"code","execution_count":153,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.83      0.72      0.75      0.77      0.74      0.54      2687\n","          1       0.62      0.75      0.72      0.68      0.74      0.54      1609\n","\n","avg / total       0.75      0.73      0.74      0.74      0.74      0.54      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, lsvm.predict(X_valid)))"]},{"cell_type":"code","execution_count":104,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[LibSVM]Accuracy: 0.7646648044692738\n","Macro-F1 score: 0.7409828058621624\n","CPU times: total: 6min 25s\n","Wall time: 6min 28s\n"]},{"data":{"text/plain":["0.7409828058621624"]},"execution_count":104,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","nu_svm = make_pipeline(NuSVC(nu=0.5, kernel=\"rbf\", gamma=\"scale\", class_weight=\"balanced\", verbose=True, random_state=100))\n","nu_svm.fit(X_train, y_train)\n","score(y_valid, nu_svm.predict(X_valid))"]},{"cell_type":"code","execution_count":155,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.85      0.62      0.82      0.73      0.54      2687\n","          1       0.72      0.62      0.85      0.66      0.73      0.51      1609\n","\n","avg / total       0.76      0.76      0.71      0.76      0.73      0.53      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, nu_svm.predict(X_valid)))"]},{"cell_type":"code","execution_count":105,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[LibSVM]Accuracy: 0.6559590316573557\n","Macro-F1 score: 0.6449642363487527\n","CPU times: total: 7min 4s\n","Wall time: 5min 33s\n"]},{"data":{"text/plain":["0.6449642363487527"]},"execution_count":105,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","svm = make_pipeline(Nystroem(kernel='cosine', gamma=0.625, coef0=0, n_components=2500, random_state=100, n_jobs=-1), SVC(C=1, kernel=\"sigmoid\", coef0=0.0, gamma=0.625, class_weight=\"balanced\", verbose=True, random_state=100))\n","svm.fit(X_train, y_train)\n","score(y_valid, svm.predict(X_valid)) # Nystroem Linear, Sigmoid, Cosine, RBF"]},{"cell_type":"code","execution_count":156,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.76      0.67      0.64      0.71      0.65      0.43      2687\n","          1       0.53      0.64      0.67      0.58      0.65      0.43      1609\n","\n","avg / total       0.67      0.66      0.65      0.66      0.65      0.43      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, svm.predict(X_valid)))"]},{"cell_type":"code","execution_count":103,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7616387337057728\n","Macro-F1 score: 0.7384961847144617\n","CPU times: total: 3min 43s\n","Wall time: 3min 44s\n"]},{"data":{"text/plain":["0.7384961847144617"]},"execution_count":103,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","ssvm = make_pipeline(SVC(C=6.89, kernel=\"rbf\", coef0=0.0, gamma=0.19, class_weight=None, verbose=0, cache_size=500, tol=0.0001, random_state=100))\n","ssvm.fit(X_train, y_train)\n","score(y_valid, ssvm.predict(X_valid))"]},{"cell_type":"code","execution_count":157,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.85      0.62      0.82      0.72      0.54      2687\n","          1       0.71      0.62      0.85      0.66      0.72      0.51      1609\n","\n","avg / total       0.76      0.76      0.70      0.76      0.72      0.53      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, ssvm.predict(X_valid)))"]},{"cell_type":"code","execution_count":158,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6897113594040968\n","Macro-F1 score: 0.6555449269886022\n","CPU times: total: 24.8 s\n","Wall time: 3.97 s\n"]},{"data":{"text/plain":["0.6555449269886022"]},"execution_count":158,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = make_pipeline_imb(KNN(n_neighbors=2, weights=\"distance\", metric=\"cosine\", n_jobs=-1))\n","knn.fit(X_train, y_train)\n","score(y_valid, knn.predict(X_valid))"]},{"cell_type":"code","execution_count":106,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7593109869646183\n","Macro-F1 score: 0.7337114312762388\n","CPU times: total: 2min 9s\n","Wall time: 2min 9s\n"]},{"data":{"text/plain":["0.7337114312762388"]},"execution_count":106,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","LogReg = LogisticRegression(penalty='l2', solver='saga', random_state=100, max_iter=500, class_weight=None, C=2.5)\n","LogReg.fit(X_train, y_train)\n","score(y_valid, LogReg.predict(X_valid))"]},{"cell_type":"code","execution_count":17,"metadata":{},"outputs":[{"ename":"NameError","evalue":"name 'LogisticRegressionCV' is not defined","output_type":"error","traceback":["\u001b[1;31m---------------------------------------------------------------------------\u001b[0m","\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)","File \u001b[1;32m<timed exec>:2\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n","\u001b[1;31mNameError\u001b[0m: name 'LogisticRegressionCV' is not defined"]}],"source":["# %%time\n","# cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)\n","# LogRegCV = LogisticRegressionCV(Cs=100, cv=cv, dual=False, penalty=\"l2\", scoring=\"f1_macro\", solver=\"saga\", n_jobs=-1, verbose=0, random_state = 100, class_weight=\"balanced\", max_iter=250)\n","# LogRegCV.fit(X_train, y_train)\n","# print(LogRegCV.coef_)\n","# print(LogRegCV.intercept_)\n","# score(y_valid, LogRegCV.predict(X_valid))"]},{"cell_type":"code","execution_count":122,"metadata":{},"outputs":[],"source":["# estimators = [('NuSVC', nu_svm), #('lsvm', lsvm) #('bayes', bayes), #('LGBM', lgbm), ('LogReg', LogReg)] # ('cbayes', cbayes), ('LogRegCV', LogRegCV), ('bayes', bayes), ('LGBM', lgbm), ('lsvm', lsvm)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)\n","# final = LogisticRegression(random_state = 100)\n","final = LGBM(boosting_type='gbdt', num_leaves=200, max_depth=150, learning_rate=0.35, colsample_bytree=0.55, subsample=0.55, min_child_samples=2, \\\n","    min_child_weight=0.1, min_split_gain=0.2, max_delta_step=0.1, n_estimators=120, class_weight=None, reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)\n","stack = StackingClassifier(estimators=estimators, final_estimator=final, cv=\"prefit\", stack_method=\"auto\", n_jobs=-1, passthrough=False, verbose=0)"]},{"cell_type":"code","execution_count":123,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7611731843575419\n","Macro-F1 score: 0.7466071880679168\n","CPU times: total: 5min 9s\n","Wall time: 5min 6s\n"]},{"data":{"text/plain":["0.7466071880679168"]},"execution_count":123,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","stack.fit(X_train, y_train)\n","score(y_valid, stack.predict(X_valid))\n","# 0.7111243089463284 LGBM: 200, LGBM\n","# 0.7137328334770291 LGBM: 200, LogReg\n","# 0.7168127964158835 LGBM: 200, LogRegBalanced\n","# 0.712740732061722 LGBM: 205, LogReg\n","# 0.7114753343602229 LGBM: 205, LGBM"]},{"cell_type":"code","execution_count":146,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.85      0.62      0.82      0.72      0.54      2687\n","          1       0.71      0.62      0.85      0.66      0.72      0.51      1609\n","\n","avg / total       0.76      0.76      0.70      0.76      0.72      0.53      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, ssvm.predict(X_valid)))"]},{"cell_type":"code","execution_count":147,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.85      0.62      0.82      0.73      0.54      2687\n","          1       0.72      0.62      0.85      0.66      0.73      0.51      1609\n","\n","avg / total       0.76      0.76      0.71      0.76      0.73      0.53      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, nu_svm.predict(X_valid)))"]},{"cell_type":"code","execution_count":148,"metadata":{},"outputs":[{"data":{"text/plain":["array([0, 0, 1, ..., 1, 0, 0], dtype=int64)"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 1min 28s\n","Wall time: 1min 30s\n"]}],"source":["%%time\n","X_pred = scaler.transform(select.transform(predict_label))\n","\n","results = nu_svm.predict(X_pred)\n","display(results)"]},{"cell_type":"code","execution_count":149,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["(4296, 1) (4296, 1)\n","\n","\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 2 columns</p>\n","</div>"],"text/plain":["         id  label\n","0     17185      0\n","1     17186      0\n","2     17187      1\n","3     17188      0\n","4     17189      0\n","...     ...    ...\n","4291  21476      0\n","4292  21477      0\n","4293  21478      1\n","4294  21479      0\n","4295  21480      0\n","\n","[4296 rows x 2 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 0 ns\n","Wall time: 12 ms\n"]},{"data":{"text/plain":["0    2929\n","1    1367\n","Name: label, dtype: int64"]},"execution_count":149,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","df_ids = df_test.iloc[:, 0].to_frame()\n","df_results = pd.DataFrame(results)\n","print(df_results.shape, df_ids.shape)\n","print(\"\\n\")\n","    \n","df_submission = pd.concat([df_ids, df_results], axis =1)\n","df_submission = df_submission.rename(columns={0: 'label'})\n","\n","display(df_submission)\n","df_submission['label'].value_counts()"]},{"cell_type":"code","execution_count":150,"metadata":{},"outputs":[],"source":["# df_submission.to_csv(f\"Stacking_Ensemble_Predictions_NuSVC_LSVM_LogReg_LGBM.csv\", index=False)\n","# df_submission.to_csv(f\"NUSVMRBF_Predictions.csv\", index=False)"]},{"cell_type":"code","execution_count":165,"metadata":{},"outputs":[],"source":["elapsed = time.time() - start_time"]},{"cell_type":"code","execution_count":166,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Process finished --- 38540.790263175964 seconds ---\n"]}],"source":["print(\"Process finished --- %s seconds ---\" % elapsed)"]},{"cell_type":"markdown","metadata":{},"source":[]},{"cell_type":"code","execution_count":167,"metadata":{},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","\n","from imblearn.combine import SMOTEENN, SMOTETomek\n","from imblearn.pipeline import Pipeline\n","\n","from lightgbm import LGBMClassifier as LGBM\n","\n","from sklearn.metrics import f1_score\n","from sklearn.decomposition import TruncatedSVD\n","from sklearn.calibration import CalibratedClassifierCV, CalibrationDisplay\n","from sklearn.model_selection import train_test_split, cross_val_score, RepeatedStratifiedKFold"]},{"cell_type":"code","execution_count":168,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>17180</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>17181</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>17182</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>17183</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>17184</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5002 columns</p>\n","</div>"],"text/plain":["          id  label    0    1    2    3    4    5    6    7  ...  4990  4991  \\\n","0          1      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1          2      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2          3      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3          4      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4          5      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...      ...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  17180      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  17181      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  17182      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  17183      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  17184      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5002 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 13.9 s\n","Wall time: 13.9 s\n"]}],"source":["%%time\n","df_train = pd.read_csv(r\"./source/train_tfidf_features.csv\")\n","display(df_train)"]},{"cell_type":"code","execution_count":169,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (15465, 5000), y_train: (15465,)\n","X_test: (1719, 5000), y_test: (1719,)\n","CPU times: total: 859 ms\n","Wall time: 386 ms\n"]}],"source":["%%time\n","X = df_train.iloc[:, 2:]\n","y = df_train.iloc[:, 1]\n","X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=100) # change test size to 0.2\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_valid.shape}, y_test: {y_valid.shape}\")"]},{"cell_type":"code","execution_count":170,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["0.9256683101733221\n"]}],"source":["svd = TruncatedSVD(n_components=3000, random_state=100)\n","svd.fit(X_train)\n","X_red = svd.transform(X_train)\n","print(np.sum(svd.explained_variance_ratio_))"]},{"cell_type":"code","execution_count":171,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["X_res: (18746, 3000), y_res: (18746,)\n"]}],"source":["smt = SMOTETomek(sampling_strategy=\"auto\", random_state=100, n_jobs=-1)\n","X_res, y_res = smt.fit_resample(X_red, y_train)\n","print(f\"X_res: {X_res.shape}, y_res: {y_res.shape}\")"]},{"cell_type":"code","execution_count":172,"metadata":{},"outputs":[],"source":["lgbm = LGBM(boosting_type='gbdt', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight='balanced', reg_lambda=0, random_state=100, verbose=100, reg_alpha=0, n_jobs=-1)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":173,"metadata":{},"outputs":[],"source":["calibrated = CalibratedClassifierCV(base_estimator=lgbm, method=\"isotonic\", cv=2, n_jobs=-1, ensemble=True)"]},{"cell_type":"code","execution_count":174,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  6.2min\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  6.3min remaining: 68.8min\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  6.3min remaining: 43.9min\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  6.4min remaining: 31.9min\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  6.5min remaining: 24.7min\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  6.5min remaining: 19.6min\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  6.5min remaining: 15.9min\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  6.6min remaining: 13.1min\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  6.6min remaining: 11.0min\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  6.6min remaining:  9.3min\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  6.6min remaining:  7.9min\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  6.9min remaining:  6.9min\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed: 12.8min remaining: 10.8min\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed: 12.8min remaining:  9.2min\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed: 12.9min remaining:  7.7min\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed: 12.9min remaining:  6.5min\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed: 13.0min remaining:  5.4min\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed: 13.2min remaining:  4.4min\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed: 13.3min remaining:  3.5min\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed: 13.4min remaining:  2.7min\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed: 13.5min remaining:  1.9min\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed: 13.6min remaining:  1.2min\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed: 13.8min remaining:    0.0s\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed: 13.8min finished\n","Macro-F1: 0.761 (0.008)\n"]}],"source":["n_scores = cross_val_score(calibrated, X_res, y_res, scoring='f1_macro', cv=cv, n_jobs=-1, verbose=100, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":175,"metadata":{},"outputs":[],"source":["X_trans = svd.transform(X_valid)\n","calibrated.fit(X_res, y_res)\n","y_pred = calibrated.predict(X_trans)"]},{"cell_type":"code","execution_count":176,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6759744037230948\n","Macro-F1 score: 0.661960747208876\n"]},{"data":{"text/plain":["0.661960747208876"]},"execution_count":176,"metadata":{},"output_type":"execute_result"}],"source":["score(y_valid, y_pred)"]},{"cell_type":"code","execution_count":177,"metadata":{},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABIwElEQVR4nO3dd3gU1dfA8e9JgoSO9E6o0kMJIEgVEOxKE5BXwYIduyJiQywoKvUnggKKBURQsaCCYkWE0EKT3kINnRBakvP+MZO4QMqmbDblfJ4nT2Zmp5zZlLMz9865oqoYY4zJuwL8HYAxxhj/skRgjDF5nCUCY4zJ4ywRGGNMHmeJwBhj8rggfweQVqVKldKQkBB/h2GMMTnKsmXLDqpq6aRey3GJICQkhPDwcH+HYYwxOYqI7EjuNbs1ZIwxeZwlAmOMyeMsERhjTB5nicAYY/I4SwTGGJPH+SwRiMgUETkgImuSeV1EZKyIbBaRCBFp6qtYjDHGJM+XVwTTgG4pvH41UMv9GgS868NYjDHGJMNniUBVfwcOp7DKjcBH6lgMFBeR8r6KxxhjcqqTJ0+yfft2n+3fn20EFYFdHvOR7rKLiMggEQkXkfCoqKgsCc4YY7KLESNG0L17d+Lj432y/xzRWKyqk1Q1TFXDSpdO8glpY4zJVY4ePcru3bsBePrppxk9ejQBAb75l+3PRLAbqOwxX8ldZowxeVpcXBytW7dmwIABABQvXpx27dr57Hj+rDU0F3hQRGYALYFjqrrXj/EYY4xfHT9+nKJFixIYGMgrr7xC5cqVU98oE/iy++hnwN/AZSISKSJ3isi9InKvu8r3wFZgMzAZuN9XsRhjTHYXERFB9erVmTt3LgA333wzYWFhWXJsn10RqGrfVF5X4AFfHd8YY3ICVUVEqFOnDjfccAM1a9bM8hhSTQQiEga0BSoAp4A1wHxVPeLj2IwxJlf77LPPGDNmDL/++ivBwcFMmTLFL3Eke2tIRAaKyHLgGaAAsAE4ALQBFojIhyJSJWvCNMaY3OfSSy+lSJEiHD9+3K9xpHRFUBC4QlVPJfWiiDTGeSp4pw/iMsaYXCc+Pp5Ro0ZRvHhxBg0aRLdu3ejatSsi4te4kr0iUNUJySUB9/WVqvqzb8IyxpjcR0T45Zdf+PPPP89b5m/p6jUkIs9ndiDGGJMbnTlzhldeeYVDhw4hIsyZM4cPP/zQ32GdJ73dR+/K1CiMMSaX2rRpEy+99BJz5swBoGDBgtniKsBTsm0EIpJc64XgNB4bY4xJQnR0NAsXLuT666+nQYMG/Pvvv1SvXt3fYSUrpSuCo0AtVS16wVcRwJ4ANsaYZCQUidu1y6mrmZ2TAKScCD4Cqibz2qc+iMUYY3KsI0eOEBkZCcAzzzzDwoULs6xEREaJ84BvzhEWFqbh4eH+DsMYYxLFxcXRsGFDKlasyPz58/0dTpJEZJmqJlmzwp9F54wxJkc7duwYxYoVIzAwkFdffZUqVXLmM7Y5YjwCY4zJbiIiIqhWrRpfffUVADfddBNNm+bModctERhjTBokjBJWt25devToQZ06dfwcUcZZIjDGGC99/PHHtGzZktOnT5MvXz4mT56cdxKBiHyb0rwxxuQFpUuXpmTJkpw4ccLfoWQqr3oNiUh5z9HDLpzPStZryBiTVeLi4njzzTcpXrw4997rjKmVMH5ATpNSryGvrggS/umLyKUi0siGlDTG5AUBAQH89ttv/P3334nLcmISSE2qiUBEfhWRoiJSAlgOTBaRt30fmjHGZL3Tp08zfPjwbF0kLrN5c0VQTFWPA92Bj1S1JdDZt2EZY4x/bN68mREjRiR2Cy1QIPeXVvMmEQSJSHmgN2CNxMaYXCc6OjrxH3+DBg3YsGEDd955p3+DykLeJILhwI/AZlVdKiLVgU2+DcsYY7LOK6+8Qq9evRJrBVWrVs3PEWUtqzVkjMmTDh8+zMmTJ6lcuTLHjh1j3bp1tGrVyt9h+UyGeg2JyBtuY3E+EflZRKJEpH/mh2mMMVkjLi6O1q1bc8cddwBQrFixXJ0EUuNN0bmrVPUpEbkZ2I7TaPw78LEvAzPGmMx29OhRihcvTmBgICNHjqRq1eQq7ectXjUWu9+vBWap6jEfxmOMMT6xatWq84rE3XjjjTRu3NivMWUX3iSCb0XkX6AZ8LOIlAZO+zYsY4zJHAlF4urVq0fv3r2pV6+enyPKflJNBKo6BGgNhKnqOeAkcKOvAzPGmIyaPn06LVq0SCwS995771G7dm1/h5XteDswTQWgs4gEeyz7yAfxGGNMpilbtixlypThxIkTBAcHp75BHpVq91EReQHoANQDvgeuBv5U1Z4+jy4J1n3UGJOcuLg4Xn/9dUqUKMF9993n73CylYwOVdkTCAVWqOpAESmL9RgyxmRDAQEB/PXXX5QvX97foeQo3iSCU6oaLyKxIlIUOABU9nFcxhjjlVOnTvH666/z0EMPUapUKebMmePVbaCwEfM5GH32ouWlCl9C+LAuvgg12/Km11C4iBQHJgPLcCqQ/p3iFi4R6SYiG0Rks4gMSeL1KiKyUERWiEiEiFyTluCNMWbr1q289tprzJ07F8DrtoCkkkBKy3OzVK8IVPV+d3KiiPwAFFXViNS2E5FAYALQBYgElorIXFVd57HaMOBzVX1XRBLaIELSeA7GmDzm+PHj/Pzzz9x8883Ur1+fTZs22cNhGZDsFYGINL3wCyiBU420qRf7boFTqG6rqp4FZnBxt1MFirrTxYA9aT8FY0xe8+qrr9K7d+/EInGWBDImpSuCt1J4TYErU9l3RWCXx3wk0PKCdV4EfhKRh4BCJDPOgYgMAgYBVKlSJZXDGmNyo0OHDnHy5EmqVKnC0KFDuemmm6hUqVKa9xMfr0xbtD3zA8zBkk0EqtoxC47fF5imqm+JSCtguog0UNX4C2KZBEwCp/toFsRljMlGEorEValShfnz51O0aFEuv/zyNO9n77FTPDFrFX9tPuSDKHMub6qPPuA2FifMXyoi96ewSYLdnN+7qJK7zNOdwOcAqvo3EAyU8mLfxpg84PDhwwAEBgYyatQo3norpRsVKft65W66vvM7K3Ye5fXuDSlV+JIk10tueW7mzQNlK1W18QXLVqhqk1S2CwI2Ap1wEsBSoJ+qrvVYZx4wU1WniUhd4GegoqYQlD1QZkzesGrVKtq3b8+UKVPo3r17uvdzLOYcw75ewzer9tC0SnHe7t2YkFKFMjHSnCGjD5QFiogk/HN2ewOlmjJVNVZEHsQZ3SwQmKKqa0VkOBCuqnOBx4HJIvIoTrvDgJSSgDEm94uLiyMwMJB69erRr18/GjVqlO59/bnpIE/MWsXB6DM8cVVt7m1fg6BAb3rN5y3eXBG8CVQF3nMX3QPsUtXHfRxbkuyKwJjc68MPP2TMmDEsWrQoQ7WBTp+L4/V5/zJt0XZqlC7E6Fua0LBSsUyMNOfJ6BXB0zg9dhIKd8wH3s+k2IwxJlGFChWoWLEi0dHR6U4Ea3Yf45GZK9l8IJoBrUMYcnUdgvMFZnKkuYuNWWyM8Zu4uDheffVVSpQowQMPPJChfcXGxfPe71t5Z/5GSha+hFG9Qmlbq3QmRZrzZfSKwBhjfCIgIIDFixdnuEjcjkMneezzVSzbcYTrGpVnxE0NKF4w7/X+SS9LBMaYLBUTE8Nrr73Gww8/nFgkLn/+/Onal6oyc+kuhn+7jsAAYUyfxtzYuGImR5z7eZ0IRKSgqsb4MhhjTO63bds23njjDapXr87AgQPTnQSiTpzhmTkRLFh/gNY1SjKqVygVihfI5GjzhlQTgYi0xmkcLgxUEZFQ4B6PYnTGGJOiY8eOMX/+fHr27En9+vXZvHkzlSunv5r9/HX7GTI7ghNnYnnuunoMbB1CQIBkYsR5izcdat8BugKHAFR1FdDOl0EZY3KX1157jX79+iUWiUtvEog+E8vTX0Rw90fhlC0azLcPteHONtUsCWSQV7eGVHWXyHlvdJxvwjHG5BZRUVHExMRQtWpVhg4dSo8ePdJVJC5B+PbDPPb5KiKPxHB/hxo80rk2lwTZw2GZwZtEsMu9PaQikg94GFjv27CMMTlZXFwcbdq0oXLlyixYsICiRYvSvHnzdO3rbGw8Y37eyLu/bqHipQWYeU8rmoeUyOSI8zZvEsG9wBicstK7gZ+AjHX4NcbkSocOHaJkyZIEBgby1ltvERISkqH9bdx/gkdnrmTtnuPcElaZ566vR+H81tkxs3nzjoqq3urzSIwxOdrKlStp3749U6dOpXv37lx33XXp3ld8vDJ10XZG/vAvRfIHMen/mnFV/XKZGK3x5E0i+EtEtgMzgdmqetSnERljcpSEInH169enf//+hIaGZmh/e446YwYs2nKIznXL8Fr3RpQukr4upsY7qba0qGptnLGF6wPLReRbEenv88iMMdne1KlTCQsL49SpU+TLl48JEyZQo0aNdO/v65W76Tr6d1bucsYMmHxbmCWBLOBVk7uqLlHVx3DGIT4MfOjTqIwxOUKVKlWoWrUqMTEZe9b0aMxZHvpsBQ/PWEmtMoWZ93Bb+rSowgW9FY2PePNAWVHgZqAPUAP4EichGGPymLi4OIYPH07p0qV58MEH6dSpE506dcrQPv/YFMWTsyI4GH2GJ7texj3tqtuYAVnMmzaCVcBXwHB3OEljTB4VEBDAsmXLqFgx4/V8PMcMqFmmMO/fHkaDinl7zAB/8SYRVLdRw4zJu06ePMkrr7zCo48+SunSpZkzZw6XXJKxyp6rI4/xyMwVbIk6ycArQni6m40Z4E/JJgIRGa2qjwBzReSiRKCqN/gyMGNM9rBjxw7efvttateuzYABAzKUBGLj4pn42xZGL9hEqcL5+fjOlrSpVSoTozXpkdIVwXT3+6isCMQYk30cPXqUn376id69e1OvXj02b96cofIQ4IwZ8OjMlSzfeZTrQysw4sYGFCuYL5MiNhmRbIuMqi5zJxur6m+eX0DjLInOGOMXr7/+Ov37908sEpeRJKCqfLZkJ1eP+YPNB6IZ06cx4/o2sSSQjXgzeP1yVW16wbIVqtrEp5Elw4aqNMY3Dhw4QExMDCEhIZw4cYKNGzfSrFmzDO0z6sQZhsyO4Od/D3BFzZK82dPGDPCXdA1VKSJ9gX5ANRGZ6/FSEZxnCYwxuURcXBxXXHEFVatWZcGCBRQpUiTDSeCntft4Zs5qTpyJ5fnr6jHAxgzItlJqI1gE7AVKAW95LD8BRPgyKGNM1oiKiqJ06dIEBgYyZsyYDBeJA2fMgJe/WcfM8F3Ur1CUGbc0plbZIhkP1vhMsolAVXcAO4BWWReOMSarrFixgnbt2jF16lR69uzJNddck+F9Lt1+mMc+X8nuI6d4oGMNHu5kYwbkBCndGvpTVduIyAnAsyFBAFXVoj6PzhiT6WJjYwkKCqJhw4bccccdGb4FBM6YAe8s2MjE37ZQ+dKCfH5PK8JszIAcI6Urgjbud7umMyaX+OCDDxg7diyLFy+mQIECjBkzJsP73Lj/BI/MWMm6vcfp07wyw66zMQNyGm9qDdUAIlX1jIh0ABoBH1k5amNynpCQEGrUqEFMTAwFCmSs9058vDLlr2288eMGiuQPYvJtYXSpVzaTIjVZyZvuoyuBMCAE+B74Gqivqhm/oZgO1n3UGO/FxcXx4osvUrp0aQYPHpxp+z1/zICyvN6jIaUKW7no7Cxd3Uc9xKtqrIjcDIxT1XEisiJzQzTG+EJAQACrVq1Kd5G4sBHzORh99qLlAhS4JJCRPRrSO6yylYvO4bxJBOfcZwpuB653l9kjgcZkU9HR0YwYMYLHHnuMMmXK8MUXX6S7PlBSSQCc3iPzHm5L1ZKFMhCpyS686dc1EKcL6Suquk1EqvFfHaIUiUg3EdkgIptFZEgy6/QWkXUislZEPvU+dGNMUnbu3Mno0aOZN28eQIYrhSbHkkDukeoVgaquE5EngNoi0gDYoKojU9tORAKBCUAXIBJYKiJzVXWdxzq1gGeAK1T1iIiUSe+JGJOXHTlyhB9//JE+ffpQr149tm7dSoUKFfwdlskhUr0icHsKbcL5p/4/YKOItPNi3y2Azaq6VVXPAjOAGy9Y525ggqoeAVDVA96HboxJMHLkSG6//XZ2794NYEnApIk3t4beAq5S1faq2g7oCrzjxXYVgV0e85HuMk+1ca40/hKRxSLSLakdicggEQkXkfCoqCgvDm1M7rd//362bdsGwLPPPsvixYszZeQwcLqGjvh2XeormlzBm8bifKq6IWFGVTeKSGY1FgcBtYAOQCXgdxFpeOEzCqo6CZgETvfRTDq2MTlWXFwcbdq0Oa9IXJMmmVMQ+PS5OB7/fBXfrd5LcL4ATp+Lv2idUoV90+5g/MObRBAuIu8DH7vztwLedOTfDVT2mK/kLvMUCfyjqueAbSKyEScxLPVi/8bkOQcOHEgsEjd27FiqVauWqfs/GnOWuz8KZ+n2Iwy7ti53tqlmXUPzAG9uDd0HrAMGu1/r3GWpWQrUEpFqInIJ0AeYe8E6X+FcDSAipXBuFW31JnBj8prly5dTvXp1vvjiCwCuvvpq6tSpk2n733U4hh7vLmLVrmOM79eEu9pWtySQR6RUdK4MMBSoCawGBqjqcW937D6E9iDwIxAITFHVtSIyHAhX1bnua1eJyDogDnhSVQ+l/3SMyX0SisQ1atSIu+66ixYtWmT6MVZHHmPgtKWci4vn47ta0qKaFYzLS5ItMSEiPwDLgN+B64DCqjowC2NLkpWYMHnJ+++/z5gxY1iyZEmGawMlZ+G/B3jg0+VcWvASPryjOTXLWJ3J3Ci9JSbKq+qz7vSPIrI880MzxqSkRo0a1KlTh1OnTvkkEXy2ZCfDvlpD3fJFmDKgOWWKBGf6MUz2l2JjsYhcilNWBCDQc15VbbhKYzJZXFwcw4YNo2zZsjzyyCN07NiRjh07ZvpxVJW3529k3C+baV+7NP+7tSmFrHR0npXST74Yzq0hz9aihKsCBar7Kihj8qqAgADWr1/P8eNeN8el2dnYeIbMiWDO8t3cElaZETc3IF+gjSKWl6U0ME1IFsZhTJ514sQJXn75ZZ544gnKlCnDrFmzyJfPN3Udj58+x/0fL+fPzQd5rEttHrqypvUMMsl3HxWRkJQ2FEelTI/ImDwmMjKScePG8cMPPwD4LAnsO3aa3hP/ZvHWQ4zqFcrgTrUsCRgg5VtDb4pIAM5ANMuAKCAYpztpR6AT8ALOQ2HGmDQ4fPgwP/zwA/369aNu3bps3bqV8uXL++x4/+47zsCpSzlxOpapA5vTtlZpnx3L5DzJXhGoai/gOeAynIJzf+AkhbuADcCVqjo/K4I0Jrd58803GThwYGKROF8mgUWbD9Lr3b+JV+Xze1pZEjAXSXWoyuzGniMwOdXevXs5deoU1atXJzo6mi1bthAaGurTY365IpKnvoigWqlCTBvYggrFffMsgsn+MjpUpTEmg+Li4mjbti0hISEsWLCAwoUL+zQJqCr/+3ULb/64gVbVSzLx/5pRrIANLGiSZonAGB/at28fZcuWJTAwkAkTJmR6kbikxMbF8/zctXz6z05ualyBkT0bkT8o0OfHNTmXdR42xkeWL19OjRo1mDVrFgBdu3aldu3aPj1mzNlY7pm+jE//2cn9HWrwdu/GlgRMqrwZoewKESnkTvcXkbdFpKrvQzMmZzp37hwAjRo14t5776VVq1ZZctyoE2foO2kxCzccYMRNDXiqWx0CAqx7qEmdN1cE7wIxIhIKPA5sAT7yaVTG5FCTJk2iSZMmxMTEEBQUxFtvvUXlypVT3zCDtkZF0/3dv9i4P5pJ/xdG/8vts5rxnjeJIFadrkU3AuNVdQJg5QmNSUKtWrWoX78+p0+fzrJjLttxmB7vLiLmTBwzBl1O53pls+zYJnfwprH4hIg8A/wf0NZ9yMy6HxiD0xto6NChlCtXjkcffdRnReKS88OavTw8YyUVihdg2sDmVC1ZKMuObXIPbxLBLUA/4A5V3SciVYA3fRuWMTlDQEAAGzdu5NSpU1l+7Cl/buPl79bRpHJx3r+9OSUK2TjCJn1SvTWkqvuA2UB+d9FB4EtfBmVMdnb8+HEef/xx9u/fj4gwa9Ysxo4dm2XHj49XXv52HcO/XUfXeuX49O7LLQmYDPGm19DdwBfAe+6iijhjDRuTJ+3Zs4d3332X+fOdCitBQVn3OM7pc3E89NkKPvhzGwNahzDh1qYE57PuoSZjvPkNfgBoAfwDoKqb3PGMjckzDh48yA8//ED//v2pU6cO27Zto2zZrG2UPRpzlrs/Cmfp9iMMu7Yud7apZtVDTabwptfQGVU9mzAjIkE4A9MYk2eMGjWKu+66iz179gBkeRLYdTiG7u8uYtWuY4zv14S72la3JGAyjTeJ4DcRGQoUEJEuwCzgG9+GZYz/7dmzh61btwIwbNgwli5dSoUKFbI8jojIo9z8v0Ucij7Lx3e15LpGWR+Dyd28SQRDcMYiWA3cA3wPDPNlUMb4W1xcHO3atWPQoEEAFC5cmIYNG2Z5HAv/PcAt7y0mf1AAs+9rRYtqJbI8BpP7edNGcBPwkapO9nEsxvjd3r17KVeuHIGBgfzvf/+jenX/Dc392ZKdDPtqDXXLF2HKgOaUKRLst1hM7ubNFcH1wEYRmS4i17ltBMbkOuHh4ecVibvqqquoWbNmlsehqoz6cQPPzFlN21qlmDmolSUB41PePEcwEGd4yllAX2CLiLzv68CMySoJReIaN27MAw88QOvWrf0Wy9nYeB7/fBXjF26mT/PKvH9bGIXy22cv41telaFW1XPAPGAGzvjFN/kwJmOyzMSJEwkNDU0sEvfmm29SqVIlv8Ry/PQ5Bk5bwpwVu3m8S21e696QoECrFG98L9WPGiJyNU6ZiQ7Ar8D7QG+fRmVMFqlbty5NmjThzJkzFCxY0G9x7D12ioFTl7L5QDRv9QqlRzP/JCOTN3lzzXkbMBO4R1XP+DgeY3wqNjaWZ555hnLlyvH444/Tvn172rdv79eY/t13nAFTlhJ9JpapA5vb4PImy6WaCFS1b1YEYkxWCAwMZPPmzYntAv721+aD3Dt9GQXzB/L5Pa2oV6Gov0MyeVCyNyBF5E/3+wkROe7xdUJEjmddiMZkzLFjx3j00UfPKxI3evRof4fFnOWRDJi6hArFC/Dl/VdYEjB+k2wiUNU27vciqlrU46uIqtpvrMkx9u7dy6RJk1iwYAGQtUXikqKqTFi4mcc+X0VY1RJ8fm8rKhQv4NeYTN7mTfXR6d4sS2bbbiKyQUQ2i8iQFNbrISIqImHe7NeY1ERFRfHRR86IqnXq1GH79u3ceuutfo4KYuPiefarNbz54wZualyBD+9oQbECNs6T8S9v+qbV95xxHyhrltpGIhIITACuBuoBfUWkXhLrFQEexq1uakxmePvttxk0aFBikbjSpf3fABtzNpZ7pi/j0392cn+HGrxzS2MuCbLuocb/UmojeEZETgCNPNsHgP3A117suwWwWVW3utVLZ+CMe3yhl4GRQNYN8mpypcjISLZs2QLAs88+y7Jly/xSJC4pUSfO0GfSYhZuOMCImxrwVLc6Vj3UZBsptRG8pqpFgDcvaB8oqarPeLHvisAuj/lId1kiEWkKVFbV71LakYgMEpFwEQmPiory4tAmr4mNjaV9+/bcc889gFMkrn79+qlslTW2REXT/d2/2LQ/mkn/F0b/y6v6OyRjzpNsq5mI1FHVf4FZ7j/s86jq8owcWEQCgLeBAamtq6qTgEkAYWFhNhaCSbR7924qVKhAUFAQ7733nl+LxCUlfPth7voonEARZgy6nNDKxf0dkjEXSan7xGPAIOCtJF5T4MpU9r0bqOwxX8ldlqAI0AD41b1ELgfMFZEbVDU8lX0bQ3h4OG3btmXq1Kn06dOHzp07+zuk88xbvZeHZ66kYvECTBvYnKolC/k7JGOSlGwiUNVB7veO6dz3UqCWiFTDSQB9gH4e+z8GlEqYF5FfgScsCZjUnD17lksuuYQmTZowePBg2rVr5++QLvLBn9sY8d06mlQuzvu3N7fB5U225k330V5uzx5EZJiIzBGRJqltp6qxwIPAj8B64HNVXSsiw0XkhowGbvKm//3vf4SGhnLy5EkCAwMZOXJktmkQBoiPV4Z/s46Xv11H13rl+PTuyy0JmGzPmydrnlPVWSLSBugMvAlMBFqmtqGqfo8zopnnsueTWbeDF7GYPK5BgwaEhYVlixIRYSPmczD6bJKvDbwihGHX1iMwwHoGmezPm07Mce73a4FJbg8f+4hjskRsbCyPPfYYo0aNAqBdu3ZMnz6d4sWL+zcwSDYJALxwfX1LAibH8OaKYLeIvAd0AUaKSH68HMfAmIwKDAxkx44d1ufeGB/y5h96b5z7/F1V9ShQAnjSl0GZvO3o0aMMHjyYffv2ISJ8/vnnvPVWUp3XjDGZwZuhKmOALUBXEXkQKKOqP/k8MpNn7d+/nylTpvDLL78AzlVBdnIuLp4pf27zdxjGZBpvRih7GLgbmOMu+lhEJqnqOJ9GZvKU/fv3M2/ePAYMGMBll13G9u3bKVWqVOobZrG/Nh/kxblr2XQg2t+hGJNpvLk1dCfQUlWfd3v8XI6TGIzJNO+88w733XdfYpG47JYEdh2O4d7py7j1/X84ExvP5NvCKFU46T4TyS03JrsS1ZQrNojIaqC5qp5254OBparaMAviu0hYWJiGh9szZ7nBrl27OHPmDDVr1iQ6Oppdu3ZRt25df4d1nlNn45j42xYm/raFABEe6FiDu9pWJzhf9rpdZUxqRGSZqiZZ6t+bXkNTgX9E5EtAcCqIfpCJ8Zk8KKFIXPXq1VmwYAGFCxfOVklAVZm3Zh+vfLee3UdPcX1oBZ65uo4NIGNyJW/GLH7bLf/QBqfG0EBVXeHrwEzuFBkZScWKFQkKCmLy5MnZrkgcwIZ9J3hx7lr+3nqIOuWKMHPQ5bSsXtLfYRnjM2kZs09wEoF16DbpsnTpUtq2bcu0adPo06cPnTp18ndI5zkWc453Fmxk+uIdFAkO4uWbGtC3eWWCAu2xGZO7edNr6HmgFzAbJwlMFZFZqjrC18GZ3OHMmTPkz5+fpk2b8thjj9G+fXt/h3SeuHhl5tJdvPnjvxw7dY5bW1blsS61udRqBJk8wpvG4g1AqEdjcQFgpapelgXxXcQai3OW8ePHM27cOJYvX06hQtmvDHP49sO8MHcta/ccp0W1Erx4fX3qVSjq77CMyXQZbSzeAwTz31CS+Tl/XAFjktWoUSNatWpFbGysv0M5z/7jp3l93r98uWI35YsFM65vE65rVN5KWZg8yZtEcAxYKyLzcdoIugBLRGQsgKoO9mF8JoeJjY3l8ccfp2LFijz11FO0a9cuW40XcCY2jil/bmfcL5uIjVMe7FiT+zvWoOAlaWkuMyZ38ea3/0v3K8GvvgnF5AZBQUHs2bOHfPny+TuUi/zy736Gf7OO7Ydi6FKvLMOurWujhhmDd91HP8yKQEzOdeTIEYYNG8Zzzz1HuXLlmDFjRraqD7Q1KpqXv13Hwg1RVC9diA/vaEH72qX9HZYx2YZdD5sMO3DgAB999BFt2rShb9++2SYJRJ+JZdwvm5jy5zbyBwUy7Nq63NYqhEuCrDuoMZ4sEZh02bdvH99//z133HFHYpG4kiWzx0NX8fHKVyt389q8f4k6cYZezSrxZLfLKFMk2N+hGZMtJfvRSESmu98fzrpwTE4xZswYHnjggcQicdklCayOPEbPiYt47PNVVCgWzJf3t+bNXqGWBIxJQbLPEYjIOpwxiucBHbjgiWJVPezr4JJizxH4z/bt2zl79iy1a9fm5MmTREZGctllfnmc5CIHo88w6scNzAzfRclCl/BUtzr0bFqJABsu0hgg/c8RTAR+BqoDyzg/Eai73OQRsbGxdOzYkerVq/Pzzz9TqFChbJEEzsXF8/HiHbw9fyOnzsZx5xXVGNy5FkWDs1+vJWOyq2QTgaqOBcaKyLuqel8WxmSykZ07d1K5cmWCgoKYMmVKtioS99fmg7z0zVo27o+mba1SvHB9PWqWKeLvsIzJcbzpPnqfiIQCbd1Fv6tqhG/DMtlBQpG4qVOn0rdvXzp27OjvkABnkJhXv1/PvDX7qFyiAJP+rxld6pW1p4KNSSdvis4NBgbx31CVn9hQlbnb6dOnCQ4OpmnTpjzxxBNceeWV/g4JOH+QGBF4vEtt7m5ng8QYk1HeFJ2LAFqp6kl3vhDwt6o2yoL4LmKNxb41duxYxo8fz4oVK7JNkbgLB4m5rlF5hl5T1waJMSYNMlp0ToA4j/k4bEyCXKtJkya0adMm2xSJ27DvBC99s5ZFW5xBYmYMupzLbZAYYzJVWoeqBLgJG6oy14iNjeWRRx6hcuXKPP3007Rt25a2bdumvqGPeQ4SUzh/EC/fWJ++LarYIDHG+EBah6oEG6oyVwkKCiIqKorChQv7OxTAGSTm8/BdvPnjBo7GnKVfyyo83uUyGyTGGB/yqsSEqi4Hlvs4FpNFDh06xLPPPsvzzz9PhQoV+OyzzwgI8P8n7WU7nEFi1uw+TouQErxwQz3qVyjm77CMyfWs1lAedOjQIT799FM6duzILbfc4vck4DlITLmiwYzt24TrbZAYY7KMTxOBiHQDxgCBwPuq+voFrz8G3AXEAlHAHaq6w5cx5VV79+7lu+++46677qJ27drs2LGDSy+91K8xJTVIzH0dalAov30+MSYr+ewvTkQCgQk4I5pFAktFZK6qrvNYbQUQpqoxInIf8AZwi69iysvGjBnD2LFjufbaaylfvrzfk4ANEmNM9uHNA2XdgZFAGZxuowKoqqY2wncLYLOqbnX3MwO4EUhMBKq60GP9xUD/NEVvUrRt2zbOnTtH7dq1ee6557jzzjspX758lsYQNmI+B6PPJvla9dKFmDawOR0uK5OlMRljzufNFcEbwPWquj6N+64I7PKYjwRaprD+nTiVTi8iIoNwnm6mSpUqaQwjb4qNjeXKK6+kRo0aLFiwgEKFClGrVq0sjyO5JADww8PtbJAYY7IBbxLB/nQkgTQRkf5AGNA+qddVdRIwCZwni30ZS063fft2qlatSlBQEFOnTqVGjRr+DilZlgSMyR68SQThIjIT+Ao4k7BQVecku4VjN1DZY76Su+w8ItIZeBZor6pnLnzdeG/JkiWJReL69etHhw4d/BLH0ZizfL1yD7OW7Up9ZWOM33mTCIoCMcBVHsuU/4rQJWcpUEtEquEkgD5AP88VRKQJ8B7QTVUPeBu0Od+pU6coUKAAzZo1Y8iQIXTu3DnLY4iLV/7YFMWsZZHMX7ufs3Hx1CufWjOSMSY78ObJ4oHp2bGqxorIg8CPON1Hp6jqWhEZDoSr6lzgTaAwMMvtM75TVW9Iz/HyqjFjxjBu3DhWrlxJ4cKFeemll7L0+FujovliWSRzlu9m3/HTFC+Yj34tq9CzWSUaVCxGyJDvsjQeY0zaedNrqBIwDrjCXfQH8LCqRqa2rap+D3x/wbLnPaaz/qNrLqGqiAhhYWFceeWVxMfHZ9mxo8/E8n3EXj4P30X4jiMECLSvXZrnr69Hp7plyB/0X1noUoUvSbLBuFRhKxlhTHbhTRnq+cCnwHR3UX/gVlXt4uPYkpTXy1DHxsYyePBgqlSpwpAhQ7LsuKrKP9sOMys8ku9X7+XUuTiqly5Er2aV6d60ImWL2uDwxmRnGS1DXVpVp3rMTxORRzIlMpNmQUFBHDlyhOLFi2fJ8XYfPcXsZZF8sSySnYdjKJw/iBsbV6BXWCWaVrnUykAYkwt4kwgOud07P3Pn+wKHfBeSudDBgwcZOnQoL774IhUqVOCTTz7xaX2g0+fi+HHtPr5YFsmfmw+iCq2ql+SRzrXo1qAcBS+xEhDG5Cbe/EXfgdNG8A5Ob6FFQLoakE36HDlyhJkzZ9K5c2d69+7tkySgqqyKPMas8F3MXbWHE6djqVi8AA9dWYtezSpRuUTBTD+mMSZ78KbX0A7AevJksd27d/Pdd98xaNAgatWqxY4dO3xyOyjqxBm+XBHJrPBINh2IJn9QAFc3KEevsMq0ql6SgAC79WNMbpdsIhCRp1T1DREZh3MlcB5VHezTyPK48ePHM3bsWK6//nrKly+fqUngXFw8v/x7gFnhkSzccIC4eKVJleK8enNDrgstT9HgfJl2LGNM9pfSFUFCWYm820Uni23ZsoVz585Rp04dnnvuOe66665MLRL3777jzAqP5KsVuzl08iyli+TnrjbV6BVWiZplimTacUz6nDt3jsjISE6fPu3vUEwOFhwcTKVKlciXz/sPdMkmAlX9xp2MUdVZnq+JSK/0hWiSExsbS6dOnahZsyYLFiygYMGCmVIn6FjMOeau2s2sZZFERB4jX6DQqU5ZeoVVon3t0jYGcDYSGRlJkSJFCAkJsd5YJl1UlUOHDhEZGUm1atW83s6bxuJngFleLDPpsG3bNkJCQggKCuLDDz/MlH/+cfHKn5sPMit8Fz+t28/Z2HjqlCvC89fV46YmFSlh4/9mS6dPn7YkYDJERChZsiRRUVFp2i6lNoKrgWuAiiIy1uOlojgjipkMWrJkCW3atGHq1KnceuuttG+fZPFVr20/eJJZy3YxZ/lu9h5zyj30bV6ZXmGVqV+hqP2DyQHsZ2QyKj2/QyldEezBaR+4AVjmsfwE8Giaj2QSeRaJe/bZZ7nqqqtS3ygZJ8/E8t3qvXwRHsmS7YcJEGhXuzTDrq1H53rnl3swxpikJHuDWFVXqeqHQEPgY1X90J3/Go9y1CZtRo8eTYMGDYiOjiYwMJAXXniB0qVLp2kfqso/Ww/xxKxVNH9lAU99EUFU9Bme6nYZi4Z0YtrAFlzbqLwlAZMmgYGBNG7cmAYNGtCrVy9iYmLStP2TTz5J/fr1efLJJ9N87FdfffW8+cKFC6d5H9568cUXGTVqFADPP/88CxYsACAkJISDBw+me78rV67k+++/T33FC3To0AF/l83xpo3gJ6AzEO3OF3CXtfZVULlRQpG4Fi1a0KVLF1Kr8ZTcEI8FLwmkdJH87DgUQ6FLArm+kVPuoVlVK/dgMqZAgQKsXLkSgFtvvZWJEyfy2GOPpbpdbGwsQUFBTJo0icOHDxMYmPYPIK+++ipDhw5N83YZNXz48DStn3CuSVm5ciXh4eFcc801mRFalvKmy0iwqiYkAdxpe8zUS7GxsQwaNIjXX38dgNatWzNx4kSKFEm5u2ZyQzzGnI2jfLFg3uoVytJhnRnZsxFhISUsCeRCHTp0YNq0aYDTtbRDhw58/PHHAMTExNChQwdmzpwJwLFjx+jQoQNz5jjDhBw8eJAOHTrwzTdO5799+/al6dht27Zl8+bNnDx5kjvuuIMWLVrQpEkTvv76awCmTZvGDTfcwJVXXkmnTp244YYbiI6OplmzZsycOZOoqCh69OhB8+bNad68OX/99RcA0dHRDBw4kIYNG9KoUSNmz57NkCFDOHXqFI0bN+bWW289L47bbruNr776KnH+1ltvTYzB08iRI2nYsCGhoaGJxRgnT55M8+bNCQ0NpUePHkle4QwYMIAvvvgicf6NN96gYcOGtGjRgs2bNyeuc++999KyZUueeuoplixZQqtWrWjSpAmtW7dmw4YNnD17lueff56ZM2fSuHFjZs6cmex7d+rUKfr06UPdunW5+eabOXXqVJp+Nr7gzRXBSRFpqqrLAUSkGeD/yHOIoKAgoqOjOXnyZKbtc8agVpm2L2MuFBsby7x58+jWrRuvvPIKV155JVOmTOHo0aO0aNEiceCj5cuXExERQYkSJQDndk7CFUW/fv149NFHadOmDTt37qRr166sX7+el19+mWLFirF69WrAKZ/So0cPxo8fn7itpzvvvJN33nmHm266iWPHjrFo0SI+/PDD89aZN28eX3/9Nf/88w8FCxbk8OHDAHTv3p27774bgGHDhvHBBx/w0EMPpXjuCbF99NFHPPLII3z77beA07V30aJFBAYGcvz4cf744w+CgoJYsGABQ4cOZfbs2QwfPpzw8HDGjx8PwNChQ5N879577z0KFizI+vXriYiIoGnTpun4KWUubxLBIzgDx+wBBCgH3OLLoHK6qKgohgwZwvDhw6lYsSKffPKJfWI3afbrr78mTufLl++8+YIFC543X6xYsfPmS5Uqdd58uXLlUj1ewqdycK4I7rzzTlq3bs3cuXMT76mfPn2anTt3AtClS5fEJHChBQsWsG7dusT548ePEx0dzYIFC5gxY0bi8ksvvTTFmNq3b8/9999PVFQUs2fPpkePHhfdmlmwYAEDBw6kYEHnRkVCTGvWrGHYsGEcPXqU6Ohounbtmup70Ldv38Tvjz76X5+YXr16Jd7yOnbsGLfffjubNm1CRDh37lyS+/rpp5+SfO9+//13Bg92CjM0atSIRo0apRqXr3lTa2ipiNQBLnMXbVDVpM/cAM4vyuzZs+nWrRu9evWyJGByBM82ggSqyuzZs7nsssvOW/7PP/9QqFChZPcVHx/P4sWLCQ7O+DgVt912Gx9//DEzZsxg6tSpqW/gGjBgAF999RWhoaFMmzbtvMSYHM+/Vc9pz3N97rnn6NixI19++SXbt29Pdmzw5N677Mjbx0ovA+oBTYG+InKb70LKmSIjI5k4cSIANWvWZMeOHfTqZQ9gm5yta9eujBs3LrFzw4oVK7za7qqrrmLcuHGJ8wkJpkuXLkyYMCFx+ZEjRwDniie5T9YDBgxg9OjRANSrV++i17t06cLUqVMT2wASbg2dOHGC8uXLc+7cOT755BOv4k5oc5k5cyatWiV9C/bYsWNUrFgRILENB6BIkSKcOHEicT65965du3Z8+umngHPVEhER4VVsvpRqIhCRF3DKUI8DOgJvYNVILzJhwgQef/xx9u7dCziX6hmR3FCONsSjyUrPPfcc586do1GjRtSvX5/nnnvOq+3Gjh1LeHg4jRo1ol69eokfkoYNG8aRI0do0KABoaGhLFy4EIBBgwbRqFGjixqLAcqWLUvdunUZODDp6vfdunXjhhtuICwsjMaNGyfeinn55Zdp2bIlV1xxBXXq1PEq7iNHjtCoUSPGjBnDO++8k+Q6Tz31FM888wxNmjQhNva/Z2s7duzIunXrEhuLk3vv7rvvPqKjo6lbty7PP/88zZo18yo2X/JmqMrVQCiwQlVDRaQsznMFeX6oys2bNxMbG0udOnWIiYlh3759VK9e3d9hmRxq/fr11K1b199hZDsxMTE0bNiQ5cuXZ/gDVl6R1O9SSkNVenNr6JSqxgOxIlIUOABUznCkOVxsbCydO3dO7IVQsGBBSwLGZLIFCxZQt25dHnroIUsCPuRNr6FwESkOTMYpNREN/O3LoLKzzZs3U6NGDYKCgpg+fXqmFIkzxiStc+fO7Nixw99h5HopXhGI02z+mqoeVdWJQBfgdlXNk0NV/vPPP9StWzex4alt27ZUqFDBz1EZY0zGpJgI1GlA+N5jfruq+r+JO4slPAzWvHlzXnjhBa6++mo/R2SMMZnHmzaC5SLS3OeRZFNvvfUWDRs25MSJEwQEBDBs2DBKlizp77CMMSbTeNNG0BLoLyLbgZM4Txerqvr/cTgfSigS16pVK7Zu3WoPhRljcq1krwhEpIo72RWoDlwJXA9c537PlWJjY7nrrrsSy+K2bt2aCRMm+LQsrjFpFTZiPiFDvrvoK2zE/Aztd9++ffTp04caNWrQrFkzrrnmGjZu3Jjs+gl/F3v27KFnz56A85DVgw8+mKE4Ro8eneYy2L/++ivXXXdd4vy8efMICwujXr16NGnShMcffxw4vwx1Zmjd+r9CzJ6luCdOnMhHH32U5v0lFfdvv/120QNusbGxlC1blj179mT4HFK6IvgKaKqqO0Rktqr2yPDRcoCgoCBOnz7NmTM25ILJvpKrTpvccm+oKjfffDO33357Yj2gVatWsX//fmrXrp3ithUqVDiviqc3x1JVAgKS/iw6evRo+vfvn1g/KK3WrFnDgw8+yHfffUedOnWIi4tj0qRJ6dpXahYtWpQ4nZFS3LGxsfz7779Jxt22bVsiIyPZsWMHVatWBZyutfXr18+UDispJQLPeyG5uoP8gQMHePrppxkxYgQVK1Zk+vTpdivI+NVL36xl3Z7j6dr2lveS7t1dr0JRXri+frLbLVy4kHz58nHvvfcmLgsNDSU6OppOnTpx5MgRzp07x4gRI7jxxhvP23b79u1cd911rFmzBoBdu3bRoUMHdu/eTf/+/XnhhRfYvn07Xbt2pWXLlixbtozvv/+e119/naVLl3Lq1Cl69uzJSy+9xNixY9mzZw8dO3akVKlSLFy4kJ9++okXXniBM2fOUKNGDaZOnUrhwoX54YcfeOSRRyhYsCBt2rRJjOeNN97g2WefTXyiODAwkPvuu++ic548eTKTJk3i7Nmz1KxZk+nTp1OwYEFmzZrFSy+9RGBgIMWKFeP3339n7dq1DBw4kLNnzxIfH8/s2bOpVasWhQsXJjo6+rxS3M888wzr16+ncOHCPPHEE2zZsoUHHniAqKgoChYsyOTJk6lTpw4DBgwgODiYFStWcMUVV3Dw4MFk4+7duzczZszg6aefBmDGjBmJRfIyKqXGYk1mOtc5fvw4X331FYsXLwZs3FiTN61ZsybJcgfBwcF8+eWXLF++nIULF/L444+nOrDSkiVLmD17NhEREcyaNStxBK5NmzZx//33s3btWqpWrcorr7xCeHg4ERER/Pbbb0RERDB48GAqVKjAwoULWbhwIQcPHmTEiBEsWLCA5cuXExYWxttvv83p06e5++67+eabb1i2bNl5Yy4kdy4X6t69O0uXLmXVqlXUrVuXDz74AHAGrPnxxx9ZtWoVc+fOBWDixIk8/PDDiQPQVKpU6bx9zZ07N7Fw3y23nF+gedCgQYwbN45ly5YxatQo7r///sTXEkpcv/322ynG3bdv38QrtTNnzvD999/To0fm3KhJ6YogVESO41wZFHCn4b/G4qKZEoGf7Ny5k2+++YYHHniAmjVrsnPnzlQHizEmq6T0yR0gZMh3yb42857MHa9CVRk6dCi///47AQEB7N69m/3796dY2rpLly6Jveu6d+/On3/+yU033UTVqlW5/PLLE9f7/PPPmTRpErGxsezdu5d169ZdVJZ58eLFrFu3jiuuuAKAs2fP0qpVK/7991+qVatGrVq1AOjfv3+ab/8kV6r6iiuuYMCAAfTu3Zvu3bsD0KpVK1555RUiIyPp3r174nFTEx0dzaJFi84rQul569mzxHVKwsLCiI6OZsOGDaxfv56WLVsmWwY8rVIaszhQVYuqahFVDXKnE+a9SgIi0k1ENojIZhEZksTr+UVkpvv6PyISkoFzSZP33nuPIUOGJH6KsCRg8rr69euzbNmyi5Z/8sknREVFsWzZMlauXEnZsmU5ffp0ivu68Ko6Yd6znPO2bdsYNWoUP//8MxEREVx77bVJ7ldV6dKlCytXrmTlypWsW7cu8ZN7Ws/lQgMGDGD8+PGsXr2aF154IfH4EydOZMSIEezatYtmzZpx6NAh+vXrl/ip/5prruGXX35Jdf/glOQuXrx4YvwrV65k/fr1ia97viepxZ1wVZCZt4XA+zLUaSYigcAE4GqcEtZ9ReTCGrJ3AkdUtSbwDjDSV/EAbNiwIXGwjGHDhrF69WqvBuwwJrvxRXXaK6+8kjNnzpz3qToiIoIdO3ZQpkwZ8uXLx8KFC70q+TB//nwOHz7MqVOn+OqrrxI/zXs6fvw4hQoVolixYuzfv5958+YlvuZZ0vnyyy/nr7/+Shw68uTJk2zcuJE6deqwfft2tmzZAsBnn32WuP2TTz7Jq6++mtjjKT4+PrECqqfkSlVv2bKFli1bMnz4cEqXLs2uXbvYunUr1atXZ/Dgwdx4441el48uWrQo1apVY9asWYCT2FatWpXkuqnF3bdvXz7++GN++eWXi9ppMsKb5wjSqwWwWVW3AojIDOBGYJ3HOjcCL7rTXwDjRUQ0tRuQ6RAbG0vXrl2pWbMmCxYsoECBAoSEhGT2YYzJEuHDMr/4r4jw5Zdf8sgjjzBy5EiCg4MJCQnhxRdfZPDgwTRs2JCwsDCvSjq3aNGCHj16EBkZSf/+/QkLC2P79u3nrRMaGkqTJk2oU6cOlStXPi9ZDBo0iG7duiW2FUybNo2+ffsm3lIZMWIEtWvXZtKkSVx77bUULFiQtm3bJiaPRo0aMXr0aPr27UtMTAwicl7X0gQJpapLly5Ny5YtE7d/8skn2bRpE6pKp06dCA0NZeTIkUyfPp18+fJRrlw5hg4d6vV7+8knn3DfffcxYsQIzp07R58+fQgNDb1ovdTirlu3LoUKFaJZs2YpDgyUVqmWoU73jkV6At1U9S53/v+Alqr6oMc6a9x1It35Le46By/Y1yBgEECVKlWapbcI1Z9//kmNGjUoX758urY3xpesDLXJLL4oQ+13qjpJVcNUNax06dLp3k+bNm0sCRhjzAV8mQh2c/64BZXcZUmuIyJBQDHgkA9jMsYYcwFfJoKlQC0RqSYilwB9gLkXrDMXuN2d7gn84ov2AWNyCvv1NxmVnt8hnyUCVY0FHgR+BNYDn6vqWhEZLiIJYx5/AJQUkc3AY8BFXUyNySuCg4M5dOiQJQOTbqrKoUOHCA4OTtN2Pmss9pXsNGaxMZnp3LlzREZGptpH35iUBAcHU6lSJfLly3fe8pQai33ZfdQYkwb58uWjWrVq/g7D5EE5oteQMcYY37FEYIwxeZwlAmOMyeNyXGOxiEQB6Xu0GEoBB1NdK3exc84b7Jzzhoycc1VVTfKJ3ByXCDJCRMKTazXPreyc8wY757zBV+dst4aMMSaPs0RgjDF5XF5LBL4ZvTp7s3POG+yc8wafnHOeaiMwxhhzsbx2RWCMMeYClgiMMSaPy5WJQES6icgGEdksIhdVNBWR/CIy0339HxEJ8UOYmcqLc35MRNaJSISI/CwiVf0RZ2ZK7Zw91ushIioiOb6roTfnLCK93Z/1WhH5NKtjzGxe/G5XEZGFIrLC/f2+xh9xZhYRmSIiB9wRHJN6XURkrPt+RIhI0wwfVFVz1RcQCGwBqgOXAKuAehescz8w0Z3uA8z0d9xZcM4dgYLu9H154Zzd9YoAvwOLgTB/x50FP+dawArgUne+jL/jzoJzngTc507XA7b7O+4MnnM7oCmwJpnXrwHmAQJcDvyT0WPmxiuCFsBmVd2qqmeBGcCNF6xzI/ChO/0F0ElEJAtjzGypnrOqLlTVGHd2Mc6IcTmZNz9ngJeBkUBuqO3szTnfDUxQ1SMAqnogi2PMbN6cswJF3eliwJ4sjC/TqervwOEUVrkR+Egdi4HiIpKhMXhzYyKoCOzymI90lyW5jjoD6BwDSmZJdL7hzTl7uhPnE0VOluo5u5fMlVX1u6wMzIe8+TnXBmqLyF8islhEumVZdL7hzTm/CPQXkUjge+ChrAnNb9L6954qG48gjxGR/kAY0N7fsfiSiAQAbwMD/BxKVgvCuT3UAeeq73cRaaiqR/0ZlI/1Baap6lsi0gqYLiINVDXe34HlFLnximA3UNljvpK7LMl1RCQI53LyUJZE5xvenDMi0hl4FrhBVc9kUWy+kto5FwEaAL+KyHace6lzc3iDsTc/50hgrqqeU9VtwEacxJBTeXPOdwKfA6jq30AwTnG23Mqrv/e0yI2JYClQS0SqicglOI3Bcy9YZy5wuzvdE/hF3VaYHCrVcxaRJsB7OEkgp983hlTOWVWPqWopVQ1R1RCcdpEbVDUnj3Pqze/2VzhXA4hIKZxbRVuzMMbM5s057wQ6AYhIXZxEEJWlUWatucBtbu+hy4Fjqro3IzvMdbeGVDVWRB4EfsTpcTBFVdeKyHAgXFXnAh/gXD5uxmmU6eO/iDPOy3N+EygMzHLbxXeq6g1+CzqDvDznXMXLc/4RuEpE1gFxwJOqmmOvdr0858eBySLyKE7D8YCc/MFORD7DSeal3HaPF4B8AKo6Eacd5BpgMxADDMzwMXPw+2WMMSYT5MZbQ8YYY9LAEoExxuRxlgiMMSaPs0RgjDF5nCUCY4zJ4ywRZDNulcyPPeaDRCRKRL71Z1xpJSLb3X7siMiiVNYdICIV0rj/kOSqM2ZEevYrIr8m9aCaiNyQUC1TRF4UkSfc6eHuw32IyCMiUjCNxxMR+UVEirrzcSKyUkTWiMisdOyvgoh84U439qze6XkOvuD5e+JLyVX0FJFRInKlr4+f3VkiyH5OAg1EpIA734UMPjWYWdynsNNMVVunssoAIE2JIKNEJNDXx1DVuar6ehLLn1fVBe7sI0Ca/nHj9CFfparH3flTqtpYVRsAZ4F70xjnHlXt6c42dvef8FqS55ADTQOSqrs0DvBZosspLBFkT98D17rTfYHPEl4QkULup5slbv31G93lISLyh4gsd79au8s7uJ9YvxCRf0Xkk6QqrbrrjPH4ZNnCXf6iiEwXkb9wHsIrLSKzRWSp+3WFu15JEflJnBr47+OUyE3Yd7TH9NMislpEVonI6yLSE6f20SfusQuISDMR+U1ElonIj+JWVnSXrxKRVcADSb1x7vn+LiLfiVPDfqI4dYcQkWgRecvdvpU4YzSscb8e8dhNkPs+rXfft4Lu9s+757xGRCZd8D7+XxLv3QARGZ9EjNNEpKeIDMZJgAvFqad/h4iM9ljvbhF5J4nTvBX4OqnzB/4AaopICRH5Spx69YtFpJG7z/ZunCvd358i7u/OGnGe3B0O3OK+fkvCOYhIMRHZ4fFeFhKRXSKST0RqiMgP7s/rDxGpk8Q5FxaRqe7PPkJEeiSxzlfuPtaKyCB3WaD7fq1xt33UXT5Y/htfY0Yy70Wi5Cp6quoOoKSIlEttH7mav2tv29dFtcajgUY45bGDgZU4Txl+677+KtDfnS6OU0umEM6nymB3eS2cpy5xtz2GU48kAPgbaJPEcX8FJrvT7XBroeNUdlwGFHDnP03YHqgCrHenxwLPu9PX4jzhWSrhnNzvVwOL+G9chBIexw5zp/O565R252/BeZoUIAJo506/SRL12t3zPY1Tvz4QmA/0dF9ToLc73QxY7b53hYG1QBMgxF3vCne9KcATnvG609OB61N57wYA4z3ex4T9TPOIabvH+1QYp/Z+Pnd+EdAwiXPcARTx/J1xvwfhJIj7cD7pvuAuvxJY6U5/43Fuhd1tQpKKOYlz+Bro6PFzed+d/hmo5U63xCnZcmHMI4HRHvOXJnH+Cb8PBYA1OBWBmwHzPbYr7n7fA+S/YFlYQkzJ/G0lnucFyycDPfz9t+/PL7siyIZUNQLnl7YvztWBp6uAISKyEucfUDDOP+R8OI/ZrwZm4QzQkWCJqkaqU41xpbvvpHzmHv93oKiIFHeXz1XVU+50Z2C8e/y57nqFcf4Bfuxu/x1wJIn9dwamqjsugqomVXP9MpxicfPdYwwDKrmxFHdjA+cfcXKWqFO/Ps49pzbu8jhgtjvdBvhSVU+qajQwB2jrvrZLVf9ypz/22L6jOCParcb551rf45jJvXdec+P4BbjO/VSdT1VXJ7FqCVU94TFfwH2vwnHq7nzgxjzd3e8vOJ96iwJ/AW+7VyPF1SnD7q2ZOAkA3AGd3J99a5zSJStx6lklVRu/MzDB41yT+v0Y7F6tLcYpqlYLp05SdREZJ05J7YTbYRE4V5H9gVh3n+GqelcazifBAbL41mR2k+tqDeUic4FROJ9wPcdKEJxPLxs8VxaRF4H9QCjOJ3/PgVg8K43GkfzP/cJ6IwnzJz2WBQCXq+p5A71I5o3rI8BaVW11wf6Lp2EfyZ3HaTc5pHl7EQkG/odz5bLLfb+DvThmWr0PDAX+BaYms06siATof2WWT6lqY88Vkvt5qOrrIvIdTjvAXyLSFe8H7ZkLvCoiJXA+qf+Cc0V19MLjp5WIdMBJFq1UNUZEfsW5wj0iIqFAV5y2j97AHThXne2A64FnxSm1nZak5ikYOJXqWrmYXRFkX1OAl5L4RPgj8FDC/WlxqoqCU0p7r/vP4f9wbouk1S3uPtvgVDQ8lsQ6P+Ex8IeINHYnfwf6ucuuBi5NYtv5wECPe+4l3OUncMpGA2wASotTVx73HnR9derpH3VjA+c+eXJaiFOtMsA9pz+TWOcP4CYRKSgihYCb3WUAVRKO757Tn/z3T/+g+ym4J+fz5r1Liue5o6r/4Hwa7odH29AFNuDc+krJH7jvkftP9qCqHheRGqq6WlVH4lT2vPB+/nnxeHKvWJYCY3BuVcap02C9TUR6uccS9x/3hebj0a4jIhf+fhQDjrhJoA5O2fCECqoBqjob5+qwqftzrayqC4Gn3W0Lp/J+pKQ2zq2oPMsSQTbl3soZm8RLL+PcBooQkbXuPDifVm93L63rcP6neG+dFpEVwEScGu9JGQyEuY106/ivh8pLQDs3pu44tyguPKcfcD5Vhru3EZ5wX5oGTHSXBeL8kx3pnstKnFsP4FRZnOCul9IlyFJgPLAe2AZ8mUQsy93jLgH+wbm3vMJ9eQPwgIisx0lo77qJaDLOP4wf3WN48ua9S8ok4AcRWeix7HPgr2RunwB8h1tqOgUvAs1EJAJ4nf/Krj/iNrxGAOe4eKS6hUC9hMbiJPY7E+jvfk9wK3Cn+/NaS9JDho4ALnWPvQpnDG1PP+A00q93413sLq+IM6bESpzbdM/g/I587N6iWwGMVdWjIhImTkeFi4hT0fNv4DIRiRSRO93l+YCaOLfV8iyrPmoAp9cQTmNmjv6DcD/9PqGq1/k5lHQT55mRd1T152ReL48zZm2XrI0s9xGRm4Gmqvqcv2PxJ7siMCabEJHiIrIR555/kkkAQJ1BSCa7jb8mY4KAt/wdhL/ZFYExxuRxdkVgjDF5nCUCY4zJ4ywRGGNMHmeJwBhj8jhLBMYYk8f9P0DSTkZ+PfWaAAAAAElFTkSuQmCC","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["disp = CalibrationDisplay.from_estimator(calibrated, X_trans, y_valid)"]},{"cell_type":"code","execution_count":178,"metadata":{},"outputs":[],"source":["from sklearn.model_selection import cross_val_score\n","from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\n","from hyperopt.pyll import scope\n","import time \n","\n","def hyperopt(param_space, model, X_train, y_train, X_test, y_test, num_eval):\n","    \n","    start = time.time()\n","    \n","    def objective_function(params):\n","        clf = model(**params)\n","        cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)\n","        n_scores = cross_val_score(clf, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, verbose=100, error_score='raise')\n","        print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))\n","        clf.fit(X_train, y_train)\n","        result = score(y_test, clf.predict(X_test))\n","        return {'loss': -result, 'status': STATUS_OK}\n","\n","    trials = Trials()\n","    best_param = fmin(objective_function, \n","                      param_space, \n","                      algo=tpe.suggest, \n","                      max_evals=num_eval, \n","                      trials=trials)\n","    loss = [x['result']['loss'] for x in trials.trials]\n","    \n","    print(\"\")\n","    print(\"##### Results\")\n","    print(\"Score best parameters: \", min(loss)*-1)\n","    print(\"Best parameters: \", best_param)\n","    print(\"Time elapsed: \", time.time() - start)\n","    print(\"Parameter combinations evaluated: \", num_eval)\n","    \n","    return trials, best_param"]},{"cell_type":"code","execution_count":179,"metadata":{},"outputs":[],"source":["LGBM_params = {\n","    'boosting_type': hp.choice('boosting_type', ['gbdt', 'dart']),\n","    'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(1)),\n","    'max_depth': scope.int(hp.quniform('max_depth', 5, 50, 1)),\n","    'num_leaves': scope.int(hp.quniform('num_leaves', 5, 50, 1)),\n","\n","    'colsample_bytree': hp.uniform('colsample_bytree', 0.4, 1.0),\n","    'subsample': hp.uniform('subsample', 0.4, 1.0),\n","\n","    'min_child_samples': scope.int(hp.quniform('min_child_samples', 10, 200, 10)),\n","    'min_child_weight': hp.loguniform('min_child_weight', np.log(1e-3), np.log(1)),\n","    'min_split_gain': hp.loguniform('min_split_gain', np.log(1e-8), np.log(1)),\n","    \n","    'max_delta_step': hp.uniform('min_delta_step', 0, 2),\n","    'n_estimators': scope.int(hp.quniform('n_estimators', 5, 100, 1)),\n","    # 'scale_pos_weight':hp.uniform('scale_pos_weight', 1.0, 5.0),\n","    # 'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),\n","    # 'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),\n","    'random_state': hp.choice('random_state', ['100']),\n","    'n_jobs': hp.choice('n_jobs', ['-1'])\n","}\n","# LGBM(boosting_type='gbdt', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight='balanced', reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)"]},{"cell_type":"code","execution_count":182,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  3.5min\n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  3.5min remaining: 38.3min\n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  3.5min remaining: 24.4min\n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  3.5min remaining: 17.5min\n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  3.5min remaining: 13.4min\n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  3.5min remaining: 10.6min\n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  3.5min remaining:  8.6min\n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  3.5min remaining:  7.1min\n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  3.5min remaining:  5.9min\n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  3.6min remaining:  5.1min\n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  3.7min remaining:  4.3min\n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  3.7min remaining:  3.7min\n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  6.8min remaining:  5.7min\n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  6.8min remaining:  4.8min\n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  6.8min remaining:  4.1min\n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  6.8min remaining:  3.4min\n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  6.9min remaining:  2.8min\n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  6.9min remaining:  2.3min\n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  6.9min remaining:  1.8min\n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  6.9min remaining:  1.4min\n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  6.9min remaining:   59.3s\n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  6.9min remaining:   37.7s\n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.0min remaining:    0.0s\n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.0min finished\n","\n","Macro-F1: 0.500 (0.007)                               \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree\n","Accuracy: 0.45782431646305993                         \n","Macro-F1 score: 0.42792317976141503                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  2.2min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  2.2min remaining: 24.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  2.2min remaining: 15.7min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.2min remaining: 11.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.2min remaining:  8.6min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.3min remaining:  6.8min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.3min remaining:  5.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.3min remaining:  4.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.3min remaining:  3.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.3min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.3min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.3min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  4.5min remaining:  3.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  4.5min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  4.6min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  4.6min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  4.6min remaining:  1.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  4.6min remaining:  1.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  4.6min remaining:  1.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.6min remaining:   55.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.6min remaining:   39.7s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.7min remaining:   25.3s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.7min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.7min finished               \n","\n","Macro-F1: 0.754 (0.009)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.6614310645724258                                                         \n","Macro-F1 score: 0.6443235856247369                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.8min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.8min remaining: 20.3min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.9min remaining: 13.0min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.9min remaining:  9.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.9min remaining:  7.1min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.9min remaining:  5.7min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.9min remaining:  4.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.9min remaining:  3.9min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.9min remaining:  3.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.0min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.0min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.5min remaining:  2.9min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.6min remaining:  2.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.7min remaining:  2.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.7min remaining:  1.8min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.7min remaining:  1.5min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.7min remaining:  1.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.7min remaining:   58.6s    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.7min remaining:   44.7s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.7min remaining:   31.9s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.8min remaining:   20.4s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.8min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.8min finished              \n","\n","Macro-F1: 0.714 (0.007)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.6218731820826061                                                        \n","Macro-F1 score: 0.612381183653646                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.9min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.9min remaining: 20.5min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.9min remaining: 13.1min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.9min remaining:  9.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.9min remaining:  7.2min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.9min remaining:  5.7min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.9min remaining:  4.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.9min remaining:  3.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.9min remaining:  3.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.0min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.0min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.7min remaining:  3.1min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.7min remaining:  2.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.7min remaining:  2.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.7min remaining:  1.9min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.7min remaining:  1.5min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.8min remaining:  1.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.8min remaining:   59.6s    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.8min remaining:   45.3s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.8min remaining:   32.5s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.8min remaining:   20.9s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.9min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.9min finished              \n","\n","Macro-F1: 0.690 (0.007)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.6259453170447935                                                        \n","Macro-F1 score: 0.6200569579243529                                                  \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.9min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.9min remaining: 21.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.9min remaining: 13.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.0min remaining:  9.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.0min remaining:  7.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.0min remaining:  5.9min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.0min remaining:  4.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.0min remaining:  4.0min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.0min remaining:  3.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.0min remaining:  2.8min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.4min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.1min remaining:  2.1min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.9min remaining:  3.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.9min remaining:  2.8min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.9min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.9min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.9min remaining:  1.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.9min remaining:  1.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.9min remaining:  1.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.0min remaining:   47.4s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.0min remaining:   34.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.0min remaining:   21.6s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.0min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.0min finished              \n","\n","Macro-F1: 0.475 (0.008)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.4316463059918557                                                        \n","Macro-F1 score: 0.38656588033114114                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.3min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.0min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.1min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.6min remaining:  4.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.6min remaining:  3.9min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.6min remaining:  3.2min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.6min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.6min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.6min remaining:  1.9min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.6min remaining:  1.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.3min remaining:  2.4min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.3min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.3min remaining:  1.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.3min remaining:  1.4min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.3min remaining:  1.1min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.3min remaining:   52.5s    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.3min remaining:   39.9s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.3min remaining:   28.5s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.3min remaining:   18.1s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min finished              \n","\n","Macro-F1: 0.363 (0.006)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.3787085514834206                                                        \n","Macro-F1 score: 0.29542691351201994                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.4min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.4min remaining: 15.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.4min remaining: 10.1min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.4min remaining:  7.3min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.5min remaining:  5.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.5min remaining:  4.4min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.5min remaining:  3.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.5min remaining:  2.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.5min remaining:  2.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.5min remaining:  2.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.5min remaining:  1.7min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.5min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.9min remaining:  2.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.9min remaining:  2.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.9min remaining:  1.7min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.9min remaining:  1.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.9min remaining:  1.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.9min remaining:   57.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.9min remaining:   45.7s  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.9min remaining:   34.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.9min remaining:   24.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.9min remaining:   15.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.9min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.9min finished            \n","\n","Macro-F1: 0.617 (0.006)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.5293775450843514                                                      \n","Macro-F1 score: 0.5248390955524873                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.      \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  2.1min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  2.1min remaining: 22.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  2.1min remaining: 14.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.1min remaining: 10.4min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.1min remaining:  7.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.1min remaining:  6.3min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.1min remaining:  5.1min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.1min remaining:  4.2min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.1min remaining:  3.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.1min remaining:  3.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.1min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.2min remaining:  2.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  4.1min remaining:  3.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  4.1min remaining:  2.9min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  4.2min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  4.2min remaining:  2.1min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  4.2min remaining:  1.7min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  4.2min remaining:  1.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  4.2min remaining:  1.1min  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.2min remaining:   50.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.2min remaining:   35.9s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.2min remaining:   22.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.2min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.2min finished            \n","\n","Macro-F1: 0.419 (0.009)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.41535776614310643                                                     \n","Macro-F1 score: 0.3617944429852414                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.      \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.5min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.5min remaining: 16.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.5min remaining: 10.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.5min remaining:  7.7min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.5min remaining:  5.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.5min remaining:  4.6min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.6min remaining:  3.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.6min remaining:  3.1min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.6min remaining:  2.6min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.6min remaining:  2.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.6min remaining:  1.9min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.6min remaining:  1.6min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.0min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.0min remaining:  2.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.1min remaining:  1.8min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.1min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.1min remaining:  1.3min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.1min remaining:  1.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.1min remaining:   48.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.1min remaining:   36.9s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.1min remaining:   26.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.1min remaining:   16.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min finished            \n","\n","Macro-F1: 0.335 (0.003)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.3624200116346713                                                      \n","Macro-F1 score: 0.2660119555935098                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.      \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.2min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.2min remaining: 13.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.2min remaining:  8.6min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.2min remaining:  6.2min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.2min remaining:  4.7min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.3min remaining:  3.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.3min remaining:  3.0min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.3min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.3min remaining:  2.1min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.3min remaining:  1.8min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.3min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.3min remaining:  1.3min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.4min remaining:  2.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.5min remaining:  1.8min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.5min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.5min remaining:  1.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.5min remaining:  1.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.5min remaining:   50.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.5min remaining:   39.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.5min remaining:   30.1s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.5min remaining:   21.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.5min remaining:   13.6s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.5min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.5min finished            \n","\n","Macro-F1: 0.500 (0.007)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.4584060500290867                                                      \n","Macro-F1 score: 0.4311922333961832                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  2.3min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  2.3min remaining: 25.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  2.4min remaining: 16.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.4min remaining: 11.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.4min remaining:  9.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.4min remaining:  7.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.4min remaining:  5.7min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.4min remaining:  4.7min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.4min remaining:  4.0min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.4min remaining:  3.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.4min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.4min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  4.6min remaining:  3.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  4.7min remaining:  3.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  4.7min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  4.7min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  4.7min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  4.7min remaining:  1.6min   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  4.7min remaining:  1.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.8min remaining:   56.9s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.8min remaining:   40.7s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.8min remaining:   25.8s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  5.1min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  5.1min finished             \n","\n","Macro-F1: 0.502 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.45782431646305993                                                      \n","Macro-F1 score: 0.4319313881781615                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.6min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.6min remaining:  4.9min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.4min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.7min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.7min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.8min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.1min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.0min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.0min remaining:  2.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.0min remaining:  1.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.0min remaining:  1.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.0min remaining:  1.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.1min remaining:  1.0min   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.1min remaining:   48.9s   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.1min remaining:   37.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.1min remaining:   26.7s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.2min remaining:   17.1s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.4min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.4min finished             \n","\n","Macro-F1: 0.647 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.5474112856311809                                                       \n","Macro-F1 score: 0.5437972787110719                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.0min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.1min remaining: 12.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.1min remaining:  7.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.2min remaining:  5.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.2min remaining:  4.4min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.2min remaining:  3.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.2min remaining:  3.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.2min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.3min remaining:  2.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.5min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.2min remaining:  2.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.3min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.3min remaining:  1.6min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.3min remaining:  1.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.4min remaining:  1.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.4min remaining:   59.8s   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.4min remaining:   48.4s   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.4min remaining:   38.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.5min remaining:   29.4s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.6min remaining:   22.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.8min remaining:   15.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min finished             \n","\n","Macro-F1: 0.424 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.41128563118091915                                                      \n","Macro-F1 score: 0.35372866848552936                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.6min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.3min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.6min remaining:  4.9min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.6min remaining:  3.9min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.6min remaining:  3.3min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.7min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.8min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.9min remaining:  2.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.0min remaining:  2.0min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.2min remaining:  2.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.2min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.2min remaining:  1.6min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.3min remaining:  1.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.3min remaining:  1.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.3min remaining:   51.7s   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.3min remaining:   39.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.3min remaining:   28.2s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.4min remaining:   18.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min finished             \n","\n","Macro-F1: 0.604 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.5287958115183246                                                       \n","Macro-F1 score: 0.5236304791099311                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.2min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.2min remaining: 13.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.2min remaining:  8.4min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.2min remaining:  6.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.2min remaining:  4.6min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.2min remaining:  3.7min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.2min remaining:  3.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.2min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.2min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.2min remaining:  1.7min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.2min remaining:  1.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.3min remaining:  1.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.2min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.3min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.3min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.3min remaining:  1.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.3min remaining:   57.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.3min remaining:   46.2s     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.3min remaining:   36.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.3min remaining:   27.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.3min remaining:   19.9s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.3min remaining:   12.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.4min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.4min finished               \n","\n","Macro-F1: 0.420 (0.007)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.40430482838859805                                                        \n","Macro-F1 score: 0.3460653720644091                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.7min remaining: 18.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.7min remaining: 11.8min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.7min remaining:  8.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.7min remaining:  6.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.7min remaining:  5.1min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.8min remaining:  2.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.8min remaining:  2.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.8min remaining:  2.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.8min remaining:  1.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.3min remaining:  2.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.3min remaining:  2.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.4min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.4min remaining:  1.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.4min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.4min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.5min remaining:   54.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.5min remaining:   41.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.5min remaining:   30.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.5min remaining:   19.2s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.6min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.6min finished               \n","\n","Macro-F1: 0.405 (0.008)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.401977894124491                                                          \n","Macro-F1 score: 0.3399406551243971                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  4.0min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  4.0min remaining: 44.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  4.0min remaining: 28.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  4.1min remaining: 20.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  4.1min remaining: 15.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  4.1min remaining: 12.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  4.1min remaining: 10.0min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  4.1min remaining:  8.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  4.1min remaining:  6.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  4.1min remaining:  5.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  4.2min remaining:  4.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  4.2min remaining:  4.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  7.6min remaining:  6.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  7.7min remaining:  5.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  7.8min remaining:  4.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  7.8min remaining:  3.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  7.8min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  7.8min remaining:  2.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  7.8min remaining:  2.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  7.8min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  7.9min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  7.9min remaining:   42.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.9min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.9min finished               \n","\n","Macro-F1: 0.644 (0.009)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.5491564863292612                                                         \n","Macro-F1 score: 0.5454126679462572                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.7min remaining:  5.0min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.0min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.7min remaining:  2.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.7min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.7min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.7min remaining:  1.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.2min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.3min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.3min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.3min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.3min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.3min remaining:   52.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.3min remaining:   39.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.3min remaining:   28.3s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.3min remaining:   18.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min finished               \n","\n","Macro-F1: 0.687 (0.008)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.5817335660267597                                                         \n","Macro-F1 score: 0.5796161120950234                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.7min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.7min remaining: 18.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.7min remaining: 11.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.7min remaining:  8.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.7min remaining:  6.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.7min remaining:  5.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.8min remaining:  2.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.8min remaining:  2.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.8min remaining:  2.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.8min remaining:  1.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.3min remaining:  2.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.4min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.4min remaining:  1.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.4min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.4min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.4min remaining:   54.3s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.5min remaining:   41.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.5min remaining:   29.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.5min remaining:   18.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min finished               \n","\n","Macro-F1: 0.426 (0.008)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.40372309482257124                                                        \n","Macro-F1 score: 0.3466658163066665                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.2min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.2min remaining: 13.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.3min remaining:  8.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.3min remaining:  6.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.3min remaining:  4.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.3min remaining:  3.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.3min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.3min remaining:  2.6min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.3min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.4min remaining:  1.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.4min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.4min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.4min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.5min remaining:  1.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.5min remaining:  1.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.5min remaining:  1.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.5min remaining:  1.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.6min remaining:   51.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.6min remaining:   40.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.6min remaining:   30.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.6min remaining:   22.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.6min remaining:   14.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.6min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.6min finished               \n","\n","Macro-F1: 0.428 (0.012)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.4083769633507853                                                         \n","Macro-F1 score: 0.35176500993549253                                                  \n","100%|██████████| 20/20 [1:24:03<00:00, 252.15s/trial, best loss: -0.6443235856247369]\n","\n","##### Results\n","Score best parameters:  0.6443235856247369\n","Best parameters:  {'boosting_type': 0, 'colsample_by_tree': 0.8360954191582883, 'learning_rate': 0.17292549690407305, 'max_depth': 34.0, 'min_child_samples': 160.0, 'n_estimators': 94.0, 'n_jobs': 0, 'num_leaves': 17.0, 'random_state': 0, 'scale_pos_weight': 1.2405835005274635, 'subsample': 0.7206617671584264}\n"]},{"ename":"NameError","evalue":"name 'clf_best' is not defined","output_type":"error","traceback":["\u001b[1;31m---------------------------------------------------------------------------\u001b[0m","\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[1;32mc:\\Users\\issac\\Desktop\\SUTD\\School Year\\Year 2\\Term 5\\50.007 Machine Learning\\Project\\work\\2022-50-007-ml-project.ipynb Cell 249\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=0'>1</a>\u001b[0m iterations, best_params \u001b[39m=\u001b[39m hyperopt(LGBM_params, LGBM,  X_res, y_res, X_trans, y_valid, \u001b[39m20\u001b[39;49m)\n","\u001b[1;32mc:\\Users\\issac\\Desktop\\SUTD\\School Year\\Year 2\\Term 5\\50.007 Machine Learning\\Project\\work\\2022-50-007-ml-project.ipynb Cell 249\u001b[0m in \u001b[0;36mhyperopt\u001b[1;34m(param_space, model, X_train, y_train, X_test, y_test, num_eval)\u001b[0m\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=28'>29</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mScore best parameters: \u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mmin\u001b[39m(loss)\u001b[39m*\u001b[39m\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=29'>30</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mBest parameters: \u001b[39m\u001b[39m\"\u001b[39m, best_param)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=30'>31</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTest Score: \u001b[39m\u001b[39m\"\u001b[39m, score(y_test, clf_best\u001b[39m.\u001b[39mpredict(X_test)))\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=31'>32</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTime elapsed: \u001b[39m\u001b[39m\"\u001b[39m, time\u001b[39m.\u001b[39mtime() \u001b[39m-\u001b[39m start)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=32'>33</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mParameter combinations evaluated: \u001b[39m\u001b[39m\"\u001b[39m, num_eval)\n","\u001b[1;31mNameError\u001b[0m: name 'clf_best' is not defined"]}],"source":["iterations, best_params = hyperopt(LGBM_params, LGBM,  X_res, y_res, X_trans, y_valid, 20)"]},{"cell_type":"code","execution_count":186,"metadata":{},"outputs":[],"source":["NuSVC_params = {\n","    'kernel': hp.choice('kernel', ['rbf', 'linear', 'sigmoid']),\n","    'nu': hp.uniform('nu', 0.1, 1.0),\n","    'gamma': hp.uniform('gamma', 1e-2, 1),\n","    'class_weight': hp.choice('class_weight', [None, 'balanced']),\n","    'max_iter': hp.choice('max_iter', [100, 200]),\n","    'random_state': hp.choice('random_state', [-1, 1000]),\n","}"]},{"cell_type":"code","execution_count":187,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 10.8min\n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed: 10.9min remaining: 120.4min\n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed: 10.9min remaining: 76.6min\n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed: 11.0min remaining: 54.8min\n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed: 11.0min remaining: 41.6min\n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed: 11.0min remaining: 32.9min\n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed: 11.0min remaining: 26.7min\n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed: 11.1min remaining: 22.1min\n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed: 11.1min remaining: 18.5min\n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed: 11.1min remaining: 15.5min\n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed: 11.1min remaining: 13.1min\n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed: 11.5min remaining: 11.5min\n","\n","  0%|          | 0/2 [11:31<?, ?trial/s, best loss=?]"]}],"source":["iterations, best_params = hyperopt(NuSVC_params, NuSVC,  X_res, y_res, X_trans, y_valid, 2)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]},{"cell_type":"markdown","metadata":{},"source":[]}],"metadata":{"kernelspec":{"display_name":"Python 3.10.5 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.5"},"vscode":{"interpreter":{"hash":"d81bf16a70e6e7c42043031d5c6c9e6d55e73e0e666fe713b30c2db86836c05c"}}},"nbformat":4,"nbformat_minor":4}