colab_notebooks/CS769_MTL_Multi_class.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":470,"status":"ok","timestamp":1651561364422,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"sqU7p0mOxQq9","outputId":"3e6f98c0-6bf9-4a0b-df1e-a1bba358545b"},"outputs":[{"name":"stdout","output_type":"stream","text":["Tue May  3 07:02:44 2022       \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n","|-------------------------------+----------------------+----------------------+\n","| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n","| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n","|                               |                      |               MIG M. |\n","|===============================+======================+======================|\n","|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n","| N/A   36C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n","|                               |                      |                  N/A |\n","+-------------------------------+----------------------+----------------------+\n","                                                                               \n","+-----------------------------------------------------------------------------+\n","| Processes:                                                                  |\n","|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n","|        ID   ID                                                   Usage      |\n","|=============================================================================|\n","|  No running processes found                                                 |\n","+-----------------------------------------------------------------------------+\n"]}],"source":["gpu_info = !nvidia-smi\n","gpu_info = '\\n'.join(gpu_info)\n","if gpu_info.find('failed') >= 0:\n","  print('Not connected to a GPU')\n","else:\n","  print(gpu_info)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":164,"status":"ok","timestamp":1651561367732,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"8OrZJmeNxVrf","outputId":"b5959ac9-b626-44f9-f4c6-a9eda21e2ac8"},"outputs":[{"name":"stdout","output_type":"stream","text":["Your runtime has 27.3 gigabytes of available RAM\n","\n","You are using a high-RAM runtime!\n"]}],"source":["from psutil import virtual_memory\n","ram_gb = virtual_memory().total / 1e9\n","print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n","\n","if ram_gb < 20:\n","  print('Not using a high-RAM runtime')\n","else:\n","  print('You are using a high-RAM runtime!')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"l_grt2dYa88l"},"outputs":[],"source":["# cd /content/drive/MyDrive/769/assignment3"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1927,"status":"ok","timestamp":1648939054595,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"qoWUB2BlatzS","outputId":"d151668d-be75-45f8-afb3-0ab39ef9b577"},"outputs":[{"name":"stdout","output_type":"stream","text":["Cloning into 'Bert-Multi-Task'...\n","remote: Enumerating objects: 58, done.\u001b[K\n","remote: Total 58 (delta 0), reused 0 (delta 0), pack-reused 58\u001b[K\n","Unpacking objects: 100% (58/58), done.\n"]}],"source":["# cd /content/drive/MyDrive/769/assignment3\n","#!git clone https://github.com/Single430/Bert-Multi-Task.git"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":157,"status":"ok","timestamp":1651561432133,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"dhakPbTPbR33","outputId":"143134a0-be49-43a6-b676-60c2c2d91ff7"},"outputs":[{"name":"stdout","output_type":"stream","text":["/content/drive/MyDrive/769/CS769_Project\n"]}],"source":["cd /content/drive/MyDrive/769/CS769_Project"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":247,"status":"ok","timestamp":1651561434396,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"_8-vFArGbWCD","outputId":"cd9c678f-de6c-44ea-da70-9279301bb260"},"outputs":[{"name":"stdout","output_type":"stream","text":["\u001b[0m\u001b[01;34mckpt\u001b[0m/                     modeling.py       run_multi_task.py\n","\u001b[01;34mcolab_notebooks\u001b[0m/          model.py          run.sh\n","\u001b[01;34mconfig\u001b[0m/                   multilabel.py     \u001b[01;34mscripts\u001b[0m/\n","\u001b[01;34mdata\u001b[0m/                     optimization.py   setup.sh\n","data_loader.py            \u001b[01;34m__pycache__\u001b[0m/      \u001b[01;34mtmp_bert\u001b[0m/\n","extract_features.py       README.md         tokenization.py\n","github.txt                report.pdf        utils.py\n","goemotions_classifier.py  requirements.txt\n"]}],"source":["ls"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"djTTxnz7be_2"},"outputs":[],"source":["#!wget https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/1502038877f6a88c225a34450793fbc3ea87eaba/download_glue_data.py"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"HVG2Gl0OlSld"},"outputs":[],"source":["# !python download_glue_data.py"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"olfYy93blVb6"},"outputs":[],"source":["# !unzip uncased_L-12_H-768_A-12.zip"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":6151,"status":"ok","timestamp":1648942871471,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"vSdXEJwemm5W","outputId":"31b4874a-248d-4099-db64-2b2dd0ba1eb7"},"outputs":[{"name":"stdout","output_type":"stream","text":["usage: convert_tf_checkpoint_to_pytorch.py [-h] --tf_checkpoint_path\n","                                           TF_CHECKPOINT_PATH\n","                                           --bert_config_file BERT_CONFIG_FILE\n","                                           --pytorch_dump_path\n","                                           PYTORCH_DUMP_PATH\n","convert_tf_checkpoint_to_pytorch.py: error: the following arguments are required: --tf_checkpoint_path, --bert_config_file, --pytorch_dump_path\n"]}],"source":[""]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":7,"status":"ok","timestamp":1649470086666,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"hNb3WTz5FA8Y","outputId":"8bf6092a-33c9-4db6-bc67-401c79faa932"},"outputs":[{"name":"stdout","output_type":"stream","text":[" \u001b[0m\u001b[01;34mconfigs\u001b[0m/                              README.md\n"," convert_tf_checkpoint_to_pytorch.py   run_multi_task.py\n"," extract_features.py                   tokenization.py\n"," \u001b[01;34mglue_data\u001b[0m/                            \u001b[01;34muncased_L-12_H-768_A-12\u001b[0m/\n"," modeling.py                          'uncased_L-12_H-768_A-12 copy.zip'\n"," optimization.py                       uncased_L-12_H-768_A-12.zip\n"]}],"source":["ls"]},{"cell_type":"markdown","metadata":{"id":"0Uut08gc8FmU"},"source":["# Multi-class /Hierarchial taxonomy tasks in MTL setting"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1871531,"status":"ok","timestamp":1651565162553,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"uRiTLmlBFBwa","outputId":"2696ae92-3b88-4768-e8e8-78c24117dedb"},"outputs":[{"name":"stdout","output_type":"stream","text":["05/03/2022 07:34:52 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 07:35:11 - INFO - __main__ -     Num Tasks = 2\n","05/03/2022 07:35:17 - INFO - __main__ -   ***** training data for goemotion_original *****\n","05/03/2022 07:35:17 - INFO - __main__ -     Data size = 36307\n","05/03/2022 07:35:24 - INFO - __main__ -   ***** training data for goemotion_ekman *****\n","05/03/2022 07:35:24 - INFO - __main__ -     Data size = 39554\n","05/03/2022 07:35:24 - INFO - __main__ -     Num param = 113148827\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 07:35:25 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 07:35:25 - INFO - __main__ -   Loss: 3.458298683166504\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 07:35:25 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 07:35:25 - INFO - __main__ -   Loss: 1.1428077220916748\n","05/03/2022 07:38:15 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:38:15 - INFO - __main__ -     eval_accuracy = 0.565207829338025\n","05/03/2022 07:38:15 - INFO - __main__ -     eval_loss = 1.6465928779230805\n","05/03/2022 07:38:15 - INFO - __main__ -     global_step = 567\n","05/03/2022 07:38:15 - INFO - __main__ -     loss = 1.4946901287351335\n","05/03/2022 07:38:26 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:38:26 - INFO - __main__ -     eval_accuracy = 0.6721941354903943\n","05/03/2022 07:38:26 - INFO - __main__ -     eval_loss = 0.9287897040510794\n","05/03/2022 07:38:26 - INFO - __main__ -     global_step = 567\n","05/03/2022 07:38:26 - INFO - __main__ -     loss = 0.4569473362263338\n","05/03/2022 07:38:26 - INFO - __main__ -   Total acc: 1.2374019648284194\n","05/03/2022 07:38:28 - INFO - __main__ -   Best Total acc: 1.2374019648284194\n","Epoch:  10% 1/10 [03:03<27:28, 183.18s/it]05/03/2022 07:38:28 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 07:38:28 - INFO - __main__ -   Loss: 1.2761950492858887\n","05/03/2022 07:38:28 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 07:38:28 - INFO - __main__ -   Loss: 0.4460790157318115\n","05/03/2022 07:41:17 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:41:17 - INFO - __main__ -     eval_accuracy = 0.6076533978447328\n","05/03/2022 07:41:17 - INFO - __main__ -     eval_loss = 1.373546871129155\n","05/03/2022 07:41:17 - INFO - __main__ -     global_step = 1134\n","05/03/2022 07:41:17 - INFO - __main__ -     loss = 0.9905928994612719\n","05/03/2022 07:41:29 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:41:29 - INFO - __main__ -     eval_accuracy = 0.7003033367037411\n","05/03/2022 07:41:29 - INFO - __main__ -     eval_loss = 0.8269395739297489\n","05/03/2022 07:41:29 - INFO - __main__ -     global_step = 1134\n","05/03/2022 07:41:29 - INFO - __main__ -     loss = 0.26817809689192124\n","05/03/2022 07:41:29 - INFO - __main__ -   Total acc: 1.307956734548474\n","05/03/2022 07:41:30 - INFO - __main__ -   Best Total acc: 1.307956734548474\n","Epoch:  20% 2/10 [06:05<24:22, 182.79s/it]05/03/2022 07:41:30 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 07:41:30 - INFO - __main__ -   Loss: 0.7383888959884644\n","05/03/2022 07:41:31 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 07:41:31 - INFO - __main__ -   Loss: 0.6298017501831055\n","05/03/2022 07:44:19 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:44:19 - INFO - __main__ -     eval_accuracy = 0.6168902573125138\n","05/03/2022 07:44:19 - INFO - __main__ -     eval_loss = 1.3133362838589244\n","05/03/2022 07:44:19 - INFO - __main__ -     global_step = 1701\n","05/03/2022 07:44:19 - INFO - __main__ -     loss = 0.780102667985139\n","05/03/2022 07:44:31 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:44:31 - INFO - __main__ -     eval_accuracy = 0.7021233569261881\n","05/03/2022 07:44:31 - INFO - __main__ -     eval_loss = 0.8004660724101044\n","05/03/2022 07:44:31 - INFO - __main__ -     global_step = 1701\n","05/03/2022 07:44:31 - INFO - __main__ -     loss = 0.2803924372271886\n","05/03/2022 07:44:31 - INFO - __main__ -   Total acc: 1.319013614238702\n","05/03/2022 07:44:32 - INFO - __main__ -   Best Total acc: 1.319013614238702\n","Epoch:  30% 3/10 [09:07<21:17, 182.44s/it]05/03/2022 07:44:32 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 07:44:32 - INFO - __main__ -   Loss: 1.7129764556884766\n","05/03/2022 07:44:33 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 07:44:33 - INFO - __main__ -   Loss: 1.5989890098571777\n","05/03/2022 07:47:21 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:47:21 - INFO - __main__ -     eval_accuracy = 0.6204090609192875\n","05/03/2022 07:47:21 - INFO - __main__ -     eval_loss = 1.2962806860151945\n","05/03/2022 07:47:21 - INFO - __main__ -     global_step = 2268\n","05/03/2022 07:47:21 - INFO - __main__ -     loss = 0.6859996594238954\n","05/03/2022 07:47:33 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:47:33 - INFO - __main__ -     eval_accuracy = 0.7130434782608696\n","05/03/2022 07:47:33 - INFO - __main__ -     eval_loss = 0.7998983214800404\n","05/03/2022 07:47:33 - INFO - __main__ -     global_step = 2268\n","05/03/2022 07:47:33 - INFO - __main__ -     loss = 0.25782148064339183\n","05/03/2022 07:47:33 - INFO - __main__ -   Total acc: 1.333452539180157\n","05/03/2022 07:47:34 - INFO - __main__ -   Best Total acc: 1.333452539180157\n","Epoch:  40% 4/10 [12:09<18:12, 182.16s/it]05/03/2022 07:47:34 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 07:47:34 - INFO - __main__ -   Loss: 1.1198036670684814\n","05/03/2022 07:47:34 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 07:47:34 - INFO - __main__ -   Loss: 1.0510386228561401\n","05/03/2022 07:50:24 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:50:24 - INFO - __main__ -     eval_accuracy = 0.622608313173521\n","05/03/2022 07:50:24 - INFO - __main__ -     eval_loss = 1.3410947671674047\n","05/03/2022 07:50:24 - INFO - __main__ -     global_step = 2835\n","05/03/2022 07:50:24 - INFO - __main__ -     loss = 0.5597729884232583\n","05/03/2022 07:50:36 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:50:36 - INFO - __main__ -     eval_accuracy = 0.7146612740141557\n","05/03/2022 07:50:36 - INFO - __main__ -     eval_loss = 0.8340634564915748\n","05/03/2022 07:50:36 - INFO - __main__ -     global_step = 2835\n","05/03/2022 07:50:36 - INFO - __main__ -     loss = 0.21527868730050545\n","05/03/2022 07:50:36 - INFO - __main__ -   Total acc: 1.3372695871876767\n","05/03/2022 07:50:37 - INFO - __main__ -   Best Total acc: 1.3372695871876767\n","Epoch:  50% 5/10 [15:12<15:12, 182.47s/it]05/03/2022 07:50:37 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 07:50:37 - INFO - __main__ -   Loss: 0.3980991244316101\n","05/03/2022 07:50:37 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 07:50:37 - INFO - __main__ -   Loss: 0.42030422389507294\n","05/03/2022 07:53:26 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:53:26 - INFO - __main__ -     eval_accuracy = 0.615790631185397\n","05/03/2022 07:53:26 - INFO - __main__ -     eval_loss = 1.3898519776984133\n","05/03/2022 07:53:26 - INFO - __main__ -     global_step = 3402\n","05/03/2022 07:53:26 - INFO - __main__ -     loss = 0.4603426588906182\n","05/03/2022 07:53:38 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:53:38 - INFO - __main__ -     eval_accuracy = 0.7083923154701719\n","05/03/2022 07:53:38 - INFO - __main__ -     eval_loss = 0.868797729498976\n","05/03/2022 07:53:38 - INFO - __main__ -     global_step = 3402\n","05/03/2022 07:53:38 - INFO - __main__ -     loss = 0.20327449073551823\n","05/03/2022 07:53:38 - INFO - __main__ -   Total acc: 1.324182946655569\n","05/03/2022 07:53:38 - INFO - __main__ -   Best Total acc: 1.3372695871876767\n","Epoch:  60% 6/10 [18:13<12:08, 182.07s/it]05/03/2022 07:53:38 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 07:53:38 - INFO - __main__ -   Loss: 0.48303699493408203\n","05/03/2022 07:53:39 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 07:53:39 - INFO - __main__ -   Loss: 0.39875170588493347\n","05/03/2022 07:56:27 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:56:27 - INFO - __main__ -     eval_accuracy = 0.6085330987464262\n","05/03/2022 07:56:27 - INFO - __main__ -     eval_loss = 1.4343378945189955\n","05/03/2022 07:56:27 - INFO - __main__ -     global_step = 3969\n","05/03/2022 07:56:27 - INFO - __main__ -     loss = 0.39670496500989116\n","05/03/2022 07:56:38 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:56:38 - INFO - __main__ -     eval_accuracy = 0.7037411526794742\n","05/03/2022 07:56:38 - INFO - __main__ -     eval_loss = 0.9328280759370481\n","05/03/2022 07:56:38 - INFO - __main__ -     global_step = 3969\n","05/03/2022 07:56:38 - INFO - __main__ -     loss = 0.1394690567433729\n","05/03/2022 07:56:38 - INFO - __main__ -   Total acc: 1.3122742514259005\n","05/03/2022 07:56:38 - INFO - __main__ -   Best Total acc: 1.3372695871876767\n","Epoch:  70% 7/10 [21:13<09:04, 181.43s/it]05/03/2022 07:56:38 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 07:56:38 - INFO - __main__ -   Loss: 0.5503250956535339\n","05/03/2022 07:56:39 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 07:56:39 - INFO - __main__ -   Loss: 0.17311207950115204\n","05/03/2022 07:59:27 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:59:27 - INFO - __main__ -     eval_accuracy = 0.6076533978447328\n","05/03/2022 07:59:27 - INFO - __main__ -     eval_loss = 1.507778225194905\n","05/03/2022 07:59:27 - INFO - __main__ -     global_step = 4536\n","05/03/2022 07:59:27 - INFO - __main__ -     loss = 0.3106830363971636\n","05/03/2022 07:59:38 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 07:59:38 - INFO - __main__ -     eval_accuracy = 0.7035389282103135\n","05/03/2022 07:59:38 - INFO - __main__ -     eval_loss = 0.993752130591619\n","05/03/2022 07:59:38 - INFO - __main__ -     global_step = 4536\n","05/03/2022 07:59:38 - INFO - __main__ -     loss = 0.09720363005391294\n","05/03/2022 07:59:38 - INFO - __main__ -   Total acc: 1.3111923260550462\n","05/03/2022 07:59:38 - INFO - __main__ -   Best Total acc: 1.3372695871876767\n","Epoch:  80% 8/10 [24:13<06:01, 180.98s/it]05/03/2022 07:59:39 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 07:59:39 - INFO - __main__ -   Loss: 0.6693382263183594\n","05/03/2022 07:59:39 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 07:59:39 - INFO - __main__ -   Loss: 0.127200186252594\n","05/03/2022 08:02:27 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:02:27 - INFO - __main__ -     eval_accuracy = 0.6094127996481197\n","05/03/2022 08:02:27 - INFO - __main__ -     eval_loss = 1.5512945115409125\n","05/03/2022 08:02:27 - INFO - __main__ -     global_step = 5103\n","05/03/2022 08:02:27 - INFO - __main__ -     loss = 0.2487925321284215\n","05/03/2022 08:02:38 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:02:38 - INFO - __main__ -     eval_accuracy = 0.700505561172902\n","05/03/2022 08:02:38 - INFO - __main__ -     eval_loss = 1.0377080930270532\n","05/03/2022 08:02:38 - INFO - __main__ -     global_step = 5103\n","05/03/2022 08:02:38 - INFO - __main__ -     loss = 0.08160521200409641\n","05/03/2022 08:02:38 - INFO - __main__ -   Total acc: 1.3099183608210216\n","05/03/2022 08:02:38 - INFO - __main__ -   Best Total acc: 1.3372695871876767\n","Epoch:  90% 9/10 [27:14<03:00, 180.72s/it]05/03/2022 08:02:39 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:02:39 - INFO - __main__ -   Loss: 0.38201436400413513\n","05/03/2022 08:02:39 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 08:02:39 - INFO - __main__ -   Loss: 0.3177252560853958\n","05/03/2022 08:05:27 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:05:27 - INFO - __main__ -     eval_accuracy = 0.609632724873543\n","05/03/2022 08:05:27 - INFO - __main__ -     eval_loss = 1.5752599217089283\n","05/03/2022 08:05:27 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:05:27 - INFO - __main__ -     loss = 0.22151197380069282\n","05/03/2022 08:05:39 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:05:39 - INFO - __main__ -     eval_accuracy = 0.6982810920121335\n","05/03/2022 08:05:39 - INFO - __main__ -     eval_loss = 1.060423914440915\n","05/03/2022 08:05:39 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:05:39 - INFO - __main__ -     loss = 0.07039672012458928\n","05/03/2022 08:05:39 - INFO - __main__ -   Total acc: 1.3079138168856765\n","05/03/2022 08:05:39 - INFO - __main__ -   Best Total acc: 1.3372695871876767\n","Epoch: 100% 10/10 [30:14<00:00, 181.44s/it]\n","05/03/2022 08:05:50 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:05:50 - INFO - __main__ -     eval_accuracy = 0.609632724873543\n","05/03/2022 08:05:50 - INFO - __main__ -     eval_loss = 1.5752599217089283\n","05/03/2022 08:05:50 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:05:50 - INFO - __main__ -     loss = 0.22151197380069282\n","05/03/2022 08:06:01 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:06:01 - INFO - __main__ -     eval_accuracy = 0.6982810920121335\n","05/03/2022 08:06:01 - INFO - __main__ -     eval_loss = 1.060423914440915\n","05/03/2022 08:06:01 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:06:01 - INFO - __main__ -     loss = 0.07039672012458928\n","05/03/2022 08:06:01 - INFO - __main__ -   Total acc: 1.3079138168856765\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/MTL_goemotion_orig_ekman \\\n","  --tasks goemotion_taxonomy \\\n","  --sample 'anneal'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1863579,"status":"ok","timestamp":1651567356399,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"2ReMHEAB8lFd","outputId":"5b5ca0c3-44aa-412c-831d-9afce589111b"},"outputs":[{"name":"stdout","output_type":"stream","text":["05/03/2022 08:11:33 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 08:11:41 - INFO - __main__ -     Num Tasks = 2\n","05/03/2022 08:11:47 - INFO - __main__ -   ***** training data for goemotion_original *****\n","05/03/2022 08:11:47 - INFO - __main__ -     Data size = 36307\n","05/03/2022 08:11:54 - INFO - __main__ -   ***** training data for goemotion_ekman *****\n","05/03/2022 08:11:54 - INFO - __main__ -     Data size = 39554\n","05/03/2022 08:11:54 - INFO - __main__ -     Num param = 113148827\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 08:11:55 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:11:55 - INFO - __main__ -   Loss: 3.458298683166504\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 08:11:55 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 08:11:55 - INFO - __main__ -   Loss: 1.1428077220916748\n","05/03/2022 08:14:45 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:14:45 - INFO - __main__ -     eval_accuracy = 0.565207829338025\n","05/03/2022 08:14:45 - INFO - __main__ -     eval_loss = 1.6465928779230805\n","05/03/2022 08:14:45 - INFO - __main__ -     global_step = 567\n","05/03/2022 08:14:45 - INFO - __main__ -     loss = 1.4946901287351335\n","05/03/2022 08:14:57 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:14:57 - INFO - __main__ -     eval_accuracy = 0.6721941354903943\n","05/03/2022 08:14:57 - INFO - __main__ -     eval_loss = 0.9287897040510794\n","05/03/2022 08:14:57 - INFO - __main__ -     global_step = 567\n","05/03/2022 08:14:57 - INFO - __main__ -     loss = 0.4569473362263338\n","05/03/2022 08:14:57 - INFO - __main__ -   Total acc: 1.2374019648284194\n","05/03/2022 08:14:59 - INFO - __main__ -   Best Total acc: 1.2374019648284194\n","Epoch:  10% 1/10 [03:04<27:38, 184.24s/it]05/03/2022 08:14:59 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:14:59 - INFO - __main__ -   Loss: 1.2761950492858887\n","05/03/2022 08:14:59 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 08:14:59 - INFO - __main__ -   Loss: 0.4460790157318115\n","05/03/2022 08:17:48 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:17:48 - INFO - __main__ -     eval_accuracy = 0.606553771717616\n","05/03/2022 08:17:48 - INFO - __main__ -     eval_loss = 1.3756053027138768\n","05/03/2022 08:17:48 - INFO - __main__ -     global_step = 1134\n","05/03/2022 08:17:48 - INFO - __main__ -     loss = 1.0017091359621213\n","05/03/2022 08:18:00 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:18:00 - INFO - __main__ -     eval_accuracy = 0.6990899898887766\n","05/03/2022 08:18:00 - INFO - __main__ -     eval_loss = 0.8411231200076844\n","05/03/2022 08:18:00 - INFO - __main__ -     global_step = 1134\n","05/03/2022 08:18:00 - INFO - __main__ -     loss = 0.26150319237767916\n","05/03/2022 08:18:00 - INFO - __main__ -   Total acc: 1.3056437616063925\n","05/03/2022 08:18:01 - INFO - __main__ -   Best Total acc: 1.3056437616063925\n","Epoch:  20% 2/10 [06:06<24:26, 183.30s/it]05/03/2022 08:18:01 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 08:18:01 - INFO - __main__ -   Loss: 0.6291160583496094\n","05/03/2022 08:18:02 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 08:18:02 - INFO - __main__ -   Loss: 1.0108507871627808\n","05/03/2022 08:20:51 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:20:51 - INFO - __main__ -     eval_accuracy = 0.615790631185397\n","05/03/2022 08:20:51 - INFO - __main__ -     eval_loss = 1.3232379466923763\n","05/03/2022 08:20:51 - INFO - __main__ -     global_step = 1701\n","05/03/2022 08:20:51 - INFO - __main__ -     loss = 0.8125685550127921\n","05/03/2022 08:21:02 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:21:02 - INFO - __main__ -     eval_accuracy = 0.7029322548028312\n","05/03/2022 08:21:02 - INFO - __main__ -     eval_loss = 0.8098688153655348\n","05/03/2022 08:21:02 - INFO - __main__ -     global_step = 1701\n","05/03/2022 08:21:02 - INFO - __main__ -     loss = 0.264299009790286\n","05/03/2022 08:21:02 - INFO - __main__ -   Total acc: 1.3187228859882283\n","05/03/2022 08:21:04 - INFO - __main__ -   Best Total acc: 1.3187228859882283\n","Epoch:  30% 3/10 [09:09<21:20, 182.92s/it]05/03/2022 08:21:04 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:21:04 - INFO - __main__ -   Loss: 1.306388020515442\n","05/03/2022 08:21:04 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 08:21:04 - INFO - __main__ -   Loss: 1.3304152488708496\n","05/03/2022 08:23:53 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:23:53 - INFO - __main__ -     eval_accuracy = 0.6208489113701342\n","05/03/2022 08:23:53 - INFO - __main__ -     eval_loss = 1.3219495745136784\n","05/03/2022 08:23:53 - INFO - __main__ -     global_step = 2268\n","05/03/2022 08:23:53 - INFO - __main__ -     loss = 0.7322393780347531\n","05/03/2022 08:24:05 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:24:05 - INFO - __main__ -     eval_accuracy = 0.7124368048533872\n","05/03/2022 08:24:05 - INFO - __main__ -     eval_loss = 0.8044319047272109\n","05/03/2022 08:24:05 - INFO - __main__ -     global_step = 2268\n","05/03/2022 08:24:05 - INFO - __main__ -     loss = 0.22813963561566838\n","05/03/2022 08:24:05 - INFO - __main__ -   Total acc: 1.3332857162235214\n","05/03/2022 08:24:06 - INFO - __main__ -   Best Total acc: 1.3332857162235214\n","Epoch:  40% 4/10 [12:12<18:16, 182.82s/it]05/03/2022 08:24:06 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:24:06 - INFO - __main__ -   Loss: 0.839348554611206\n","05/03/2022 08:24:07 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 08:24:07 - INFO - __main__ -   Loss: 0.7686742544174194\n","05/03/2022 08:26:56 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:26:56 - INFO - __main__ -     eval_accuracy = 0.6256872663294479\n","05/03/2022 08:26:56 - INFO - __main__ -     eval_loss = 1.3351326279983788\n","05/03/2022 08:26:56 - INFO - __main__ -     global_step = 2835\n","05/03/2022 08:26:56 - INFO - __main__ -     loss = 0.6144148272706929\n","05/03/2022 08:27:07 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:27:07 - INFO - __main__ -     eval_accuracy = 0.714459049544995\n","05/03/2022 08:27:07 - INFO - __main__ -     eval_loss = 0.8359032244705904\n","05/03/2022 08:27:07 - INFO - __main__ -     global_step = 2835\n","05/03/2022 08:27:07 - INFO - __main__ -     loss = 0.18035910064370964\n","05/03/2022 08:27:07 - INFO - __main__ -   Total acc: 1.3401463158744429\n","05/03/2022 08:27:09 - INFO - __main__ -   Best Total acc: 1.3401463158744429\n","Epoch:  50% 5/10 [15:14<15:13, 182.67s/it]05/03/2022 08:27:09 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 08:27:09 - INFO - __main__ -   Loss: 0.44714298844337463\n","05/03/2022 08:27:09 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 08:27:09 - INFO - __main__ -   Loss: 0.3440137207508087\n","05/03/2022 08:29:58 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:29:58 - INFO - __main__ -     eval_accuracy = 0.6052342203650759\n","05/03/2022 08:29:58 - INFO - __main__ -     eval_loss = 1.3766645604256795\n","05/03/2022 08:29:58 - INFO - __main__ -     global_step = 3402\n","05/03/2022 08:29:58 - INFO - __main__ -     loss = 0.5028939301269605\n","05/03/2022 08:30:10 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:30:10 - INFO - __main__ -     eval_accuracy = 0.6984833164812942\n","05/03/2022 08:30:10 - INFO - __main__ -     eval_loss = 0.86468958852268\n","05/03/2022 08:30:10 - INFO - __main__ -     global_step = 3402\n","05/03/2022 08:30:10 - INFO - __main__ -     loss = 0.17353452889271725\n","05/03/2022 08:30:10 - INFO - __main__ -   Total acc: 1.3037175368463703\n","05/03/2022 08:30:10 - INFO - __main__ -   Best Total acc: 1.3401463158744429\n","Epoch:  60% 6/10 [18:15<12:08, 182.22s/it]05/03/2022 08:30:10 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 08:30:10 - INFO - __main__ -   Loss: 0.222349613904953\n","05/03/2022 08:30:11 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 08:30:11 - INFO - __main__ -   Loss: 0.29173439741134644\n","05/03/2022 08:32:59 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:32:59 - INFO - __main__ -     eval_accuracy = 0.6162304816362437\n","05/03/2022 08:32:59 - INFO - __main__ -     eval_loss = 1.4417566817731766\n","05/03/2022 08:32:59 - INFO - __main__ -     global_step = 3969\n","05/03/2022 08:32:59 - INFO - __main__ -     loss = 0.44970040015442664\n","05/03/2022 08:33:11 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:33:11 - INFO - __main__ -     eval_accuracy = 0.7053589484327604\n","05/03/2022 08:33:11 - INFO - __main__ -     eval_loss = 0.923820919316617\n","05/03/2022 08:33:11 - INFO - __main__ -     global_step = 3969\n","05/03/2022 08:33:11 - INFO - __main__ -     loss = 0.12489715309212447\n","05/03/2022 08:33:11 - INFO - __main__ -   Total acc: 1.321589430069004\n","05/03/2022 08:33:11 - INFO - __main__ -   Best Total acc: 1.3401463158744429\n","Epoch:  70% 7/10 [21:16<09:05, 181.67s/it]05/03/2022 08:33:11 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:33:11 - INFO - __main__ -   Loss: 0.6851995587348938\n","05/03/2022 08:33:11 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 08:33:11 - INFO - __main__ -   Loss: 0.21886256337165833\n","05/03/2022 08:36:00 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:36:00 - INFO - __main__ -     eval_accuracy = 0.6102925005498131\n","05/03/2022 08:36:00 - INFO - __main__ -     eval_loss = 1.5070401814705667\n","05/03/2022 08:36:00 - INFO - __main__ -     global_step = 4536\n","05/03/2022 08:36:00 - INFO - __main__ -     loss = 0.34489470733431493\n","05/03/2022 08:36:11 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:36:11 - INFO - __main__ -     eval_accuracy = 0.7031344792719919\n","05/03/2022 08:36:11 - INFO - __main__ -     eval_loss = 0.9717007311683913\n","05/03/2022 08:36:11 - INFO - __main__ -     global_step = 4536\n","05/03/2022 08:36:11 - INFO - __main__ -     loss = 0.08016775022843949\n","05/03/2022 08:36:11 - INFO - __main__ -   Total acc: 1.313426979821805\n","05/03/2022 08:36:11 - INFO - __main__ -   Best Total acc: 1.3401463158744429\n","Epoch:  80% 8/10 [24:17<06:02, 181.39s/it]05/03/2022 08:36:12 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:36:12 - INFO - __main__ -   Loss: 0.5100865364074707\n","05/03/2022 08:36:12 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 08:36:12 - INFO - __main__ -   Loss: 0.4771921932697296\n","05/03/2022 08:39:00 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:39:00 - INFO - __main__ -     eval_accuracy = 0.6083131735210029\n","05/03/2022 08:39:00 - INFO - __main__ -     eval_loss = 1.5589699259122771\n","05/03/2022 08:39:00 - INFO - __main__ -     global_step = 5103\n","05/03/2022 08:39:00 - INFO - __main__ -     loss = 0.2846613795852955\n","05/03/2022 08:39:12 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:39:12 - INFO - __main__ -     eval_accuracy = 0.6982810920121335\n","05/03/2022 08:39:12 - INFO - __main__ -     eval_loss = 1.022452809473256\n","05/03/2022 08:39:12 - INFO - __main__ -     global_step = 5103\n","05/03/2022 08:39:12 - INFO - __main__ -     loss = 0.06449228405584524\n","05/03/2022 08:39:12 - INFO - __main__ -   Total acc: 1.3065942655331364\n","05/03/2022 08:39:12 - INFO - __main__ -   Best Total acc: 1.3401463158744429\n","Epoch:  90% 9/10 [27:17<03:01, 181.17s/it]05/03/2022 08:39:12 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:39:12 - INFO - __main__ -   Loss: 0.2137993574142456\n","05/03/2022 08:39:13 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 08:39:13 - INFO - __main__ -   Loss: 0.32757629454135895\n","05/03/2022 08:42:01 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:42:01 - INFO - __main__ -     eval_accuracy = 0.6023751924345723\n","05/03/2022 08:42:01 - INFO - __main__ -     eval_loss = 1.5822609359879696\n","05/03/2022 08:42:01 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:42:01 - INFO - __main__ -     loss = 0.26967512464029236\n","05/03/2022 08:42:13 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:42:13 - INFO - __main__ -     eval_accuracy = 0.6996966632962589\n","05/03/2022 08:42:13 - INFO - __main__ -     eval_loss = 1.0385191991522018\n","05/03/2022 08:42:13 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:42:13 - INFO - __main__ -     loss = 0.05061865298363267\n","05/03/2022 08:42:13 - INFO - __main__ -   Total acc: 1.3020718557308313\n","05/03/2022 08:42:13 - INFO - __main__ -   Best Total acc: 1.3401463158744429\n","Epoch: 100% 10/10 [30:18<00:00, 181.83s/it]\n","05/03/2022 08:42:23 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:42:23 - INFO - __main__ -     eval_accuracy = 0.6023751924345723\n","05/03/2022 08:42:23 - INFO - __main__ -     eval_loss = 1.5822609359879696\n","05/03/2022 08:42:23 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:42:23 - INFO - __main__ -     loss = 0.26967512464029236\n","05/03/2022 08:42:35 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:42:35 - INFO - __main__ -     eval_accuracy = 0.6996966632962589\n","05/03/2022 08:42:35 - INFO - __main__ -     eval_loss = 1.0385191991522018\n","05/03/2022 08:42:35 - INFO - __main__ -     global_step = 5670\n","05/03/2022 08:42:35 - INFO - __main__ -     loss = 0.05061865298363267\n","05/03/2022 08:42:35 - INFO - __main__ -   Total acc: 1.3020718557308313\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/MTL_goemotion_orig_ekman_prop \\\n","  --tasks goemotion_taxonomy \\\n","  --sample 'prop'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1626423,"status":"ok","timestamp":1651570045744,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"RLzfJgwY8mwe","outputId":"cf980469-e888-41a8-d093-da176b1e306e"},"outputs":[{"name":"stdout","output_type":"stream","text":["05/03/2022 08:56:29 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 08:56:36 - INFO - __main__ -     Num Tasks = 2\n","05/03/2022 08:56:43 - INFO - __main__ -   ***** training data for goemotion_original *****\n","05/03/2022 08:56:43 - INFO - __main__ -     Data size = 36307\n","05/03/2022 08:56:50 - INFO - __main__ -   ***** training data for goemotion_ekman *****\n","05/03/2022 08:56:50 - INFO - __main__ -     Data size = 39554\n","05/03/2022 08:56:50 - INFO - __main__ -     Num param = 113148827\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 08:56:50 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:56:50 - INFO - __main__ -   Loss: 3.458298683166504\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 08:56:50 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 08:56:50 - INFO - __main__ -   Loss: 1.1428077220916748\n","05/03/2022 08:59:40 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:59:40 - INFO - __main__ -     eval_accuracy = 0.5601495491532879\n","05/03/2022 08:59:40 - INFO - __main__ -     eval_loss = 1.6617658184366613\n","05/03/2022 08:59:40 - INFO - __main__ -     global_step = 567\n","05/03/2022 08:59:40 - INFO - __main__ -     loss = 1.3112337601878656\n","05/03/2022 08:59:52 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 08:59:52 - INFO - __main__ -     eval_accuracy = 0.679474216380182\n","05/03/2022 08:59:52 - INFO - __main__ -     eval_loss = 0.8975400461759628\n","05/03/2022 08:59:52 - INFO - __main__ -     global_step = 567\n","05/03/2022 08:59:52 - INFO - __main__ -     loss = 0.5580587170531931\n","05/03/2022 08:59:52 - INFO - __main__ -   Total acc: 1.2396237655334699\n","05/03/2022 08:59:53 - INFO - __main__ -   Best Total acc: 1.2396237655334699\n","Epoch:  10% 1/10 [03:03<27:29, 183.29s/it]05/03/2022 08:59:53 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 08:59:53 - INFO - __main__ -   Loss: 1.2641946077346802\n","05/03/2022 08:59:54 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 08:59:54 - INFO - __main__ -   Loss: 0.32687658071517944\n","05/03/2022 09:02:43 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:02:43 - INFO - __main__ -     eval_accuracy = 0.6023751924345723\n","05/03/2022 09:02:43 - INFO - __main__ -     eval_loss = 1.4070349949524985\n","05/03/2022 09:02:43 - INFO - __main__ -     global_step = 1134\n","05/03/2022 09:02:43 - INFO - __main__ -     loss = 0.8972363271200258\n","05/03/2022 09:02:54 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:02:54 - INFO - __main__ -     eval_accuracy = 0.7003033367037411\n","05/03/2022 09:02:54 - INFO - __main__ -     eval_loss = 0.8402277004027887\n","05/03/2022 09:02:54 - INFO - __main__ -     global_step = 1134\n","05/03/2022 09:02:54 - INFO - __main__ -     loss = 0.3224360873985122\n","05/03/2022 09:02:54 - INFO - __main__ -   Total acc: 1.3026785291383134\n","05/03/2022 09:02:56 - INFO - __main__ -   Best Total acc: 1.3026785291383134\n","Epoch:  20% 2/10 [06:05<24:22, 182.87s/it]05/03/2022 09:02:56 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 09:02:56 - INFO - __main__ -   Loss: 0.9280919432640076\n","05/03/2022 09:02:56 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 09:02:56 - INFO - __main__ -   Loss: 0.8691309690475464\n","05/03/2022 09:05:45 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:05:45 - INFO - __main__ -     eval_accuracy = 0.6190895095667474\n","05/03/2022 09:05:45 - INFO - __main__ -     eval_loss = 1.3400835915745248\n","05/03/2022 09:05:45 - INFO - __main__ -     global_step = 1701\n","05/03/2022 09:05:45 - INFO - __main__ -     loss = 0.7161124816437037\n","05/03/2022 09:05:57 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:05:57 - INFO - __main__ -     eval_accuracy = 0.7073811931243681\n","05/03/2022 09:05:57 - INFO - __main__ -     eval_loss = 0.8033604539557689\n","05/03/2022 09:05:57 - INFO - __main__ -     global_step = 1701\n","05/03/2022 09:05:57 - INFO - __main__ -     loss = 0.31635382634832326\n","05/03/2022 09:05:57 - INFO - __main__ -   Total acc: 1.3264707026911156\n","05/03/2022 09:05:58 - INFO - __main__ -   Best Total acc: 1.3264707026911156\n","Epoch:  30% 3/10 [09:08<21:18, 182.63s/it]05/03/2022 09:05:58 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 09:05:58 - INFO - __main__ -   Loss: 1.0453941822052002\n","05/03/2022 09:05:59 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 09:05:59 - INFO - __main__ -   Loss: 1.333738088607788\n","05/03/2022 09:08:47 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:08:47 - INFO - __main__ -     eval_accuracy = 0.6265669672311414\n","05/03/2022 09:08:47 - INFO - __main__ -     eval_loss = 1.3059359716352972\n","05/03/2022 09:08:47 - INFO - __main__ -     global_step = 2268\n","05/03/2022 09:08:47 - INFO - __main__ -     loss = 0.6659070872545663\n","05/03/2022 09:08:59 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:08:59 - INFO - __main__ -     eval_accuracy = 0.7201213346814964\n","05/03/2022 09:08:59 - INFO - __main__ -     eval_loss = 0.8042899882947493\n","05/03/2022 09:08:59 - INFO - __main__ -     global_step = 2268\n","05/03/2022 09:08:59 - INFO - __main__ -     loss = 0.29070734168276374\n","05/03/2022 09:08:59 - INFO - __main__ -   Total acc: 1.3466883019126379\n","05/03/2022 09:09:00 - INFO - __main__ -   Best Total acc: 1.3466883019126379\n","Epoch:  40% 4/10 [12:10<18:14, 182.38s/it]05/03/2022 09:09:00 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 09:09:00 - INFO - __main__ -   Loss: 1.1697185039520264\n","05/03/2022 09:09:01 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 09:09:01 - INFO - __main__ -   Loss: 1.0658989250659943\n","05/03/2022 09:11:49 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:11:49 - INFO - __main__ -     eval_accuracy = 0.6201891356938641\n","05/03/2022 09:11:49 - INFO - __main__ -     eval_loss = 1.356605665702602\n","05/03/2022 09:11:49 - INFO - __main__ -     global_step = 2835\n","05/03/2022 09:11:49 - INFO - __main__ -     loss = 0.5663953266867043\n","05/03/2022 09:12:01 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:12:01 - INFO - __main__ -     eval_accuracy = 0.7069767441860465\n","05/03/2022 09:12:01 - INFO - __main__ -     eval_loss = 0.8336539176835863\n","05/03/2022 09:12:01 - INFO - __main__ -     global_step = 2835\n","05/03/2022 09:12:01 - INFO - __main__ -     loss = 0.23117420710688966\n","05/03/2022 09:12:01 - INFO - __main__ -   Total acc: 1.3271658798799106\n","05/03/2022 09:12:01 - INFO - __main__ -   Best Total acc: 1.3466883019126379\n","Epoch:  50% 5/10 [15:11<15:09, 181.85s/it]05/03/2022 09:12:01 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 09:12:01 - INFO - __main__ -   Loss: 0.6497591137886047\n","05/03/2022 09:12:01 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 09:12:01 - INFO - __main__ -   Loss: 0.5463612228631973\n","05/03/2022 09:14:50 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:14:50 - INFO - __main__ -     eval_accuracy = 0.6144710798328569\n","05/03/2022 09:14:50 - INFO - __main__ -     eval_loss = 1.4072423258334765\n","05/03/2022 09:14:50 - INFO - __main__ -     global_step = 3402\n","05/03/2022 09:14:50 - INFO - __main__ -     loss = 0.44964085343233073\n","05/03/2022 09:15:01 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:15:01 - INFO - __main__ -     eval_accuracy = 0.7041456016177957\n","05/03/2022 09:15:01 - INFO - __main__ -     eval_loss = 0.880968565220594\n","05/03/2022 09:15:01 - INFO - __main__ -     global_step = 3402\n","05/03/2022 09:15:01 - INFO - __main__ -     loss = 0.1997173711053068\n","05/03/2022 09:15:01 - INFO - __main__ -   Total acc: 1.3186166814506526\n","05/03/2022 09:15:01 - INFO - __main__ -   Best Total acc: 1.3466883019126379\n","Epoch:  60% 6/10 [18:11<12:05, 181.32s/it]05/03/2022 09:15:01 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 09:15:01 - INFO - __main__ -   Loss: 0.3024017810821533\n","05/03/2022 09:15:02 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 09:15:02 - INFO - __main__ -   Loss: 0.25957438349723816\n","05/03/2022 09:17:50 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:17:50 - INFO - __main__ -     eval_accuracy = 0.6168902573125138\n","05/03/2022 09:17:50 - INFO - __main__ -     eval_loss = 1.4566890633483345\n","05/03/2022 09:17:50 - INFO - __main__ -     global_step = 3969\n","05/03/2022 09:17:50 - INFO - __main__ -     loss = 0.4031967947512497\n","05/03/2022 09:18:02 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:18:02 - INFO - __main__ -     eval_accuracy = 0.7061678463094034\n","05/03/2022 09:18:02 - INFO - __main__ -     eval_loss = 0.9535065335830936\n","05/03/2022 09:18:02 - INFO - __main__ -     global_step = 3969\n","05/03/2022 09:18:02 - INFO - __main__ -     loss = 0.12409548356314395\n","05/03/2022 09:18:02 - INFO - __main__ -   Total acc: 1.3230581036219173\n","05/03/2022 09:18:02 - INFO - __main__ -   Best Total acc: 1.3466883019126379\n","Epoch:  70% 7/10 [21:11<09:03, 181.06s/it]05/03/2022 09:18:02 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 09:18:02 - INFO - __main__ -   Loss: 0.30503562092781067\n","05/03/2022 09:18:02 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 09:18:02 - INFO - __main__ -   Loss: 0.11342934519052505\n","05/03/2022 09:20:50 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:20:50 - INFO - __main__ -     eval_accuracy = 0.6078733230701562\n","05/03/2022 09:20:50 - INFO - __main__ -     eval_loss = 1.521690357417549\n","05/03/2022 09:20:50 - INFO - __main__ -     global_step = 4536\n","05/03/2022 09:20:50 - INFO - __main__ -     loss = 0.3491537418525265\n","05/03/2022 09:21:02 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:21:02 - INFO - __main__ -     eval_accuracy = 0.7021233569261881\n","05/03/2022 09:21:02 - INFO - __main__ -     eval_loss = 1.0066870168748165\n","05/03/2022 09:21:02 - INFO - __main__ -     global_step = 4536\n","05/03/2022 09:21:02 - INFO - __main__ -     loss = 0.0901900380652733\n","05/03/2022 09:21:02 - INFO - __main__ -   Total acc: 1.3099966799963443\n","05/03/2022 09:21:02 - INFO - __main__ -   Best Total acc: 1.3466883019126379\n","Epoch:  80% 8/10 [24:11<06:01, 180.73s/it]05/03/2022 09:21:02 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 09:21:02 - INFO - __main__ -   Loss: 0.5283353924751282\n","05/03/2022 09:21:02 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 09:21:02 - INFO - __main__ -   Loss: 0.12547487020492554\n","05/03/2022 09:23:50 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:23:50 - INFO - __main__ -     eval_accuracy = 0.6023751924345723\n","05/03/2022 09:23:50 - INFO - __main__ -     eval_loss = 1.5650238297709471\n","05/03/2022 09:23:50 - INFO - __main__ -     global_step = 5103\n","05/03/2022 09:23:50 - INFO - __main__ -     loss = 0.27060795242929375\n","05/03/2022 09:24:02 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:24:02 - INFO - __main__ -     eval_accuracy = 0.6956521739130435\n","05/03/2022 09:24:02 - INFO - __main__ -     eval_loss = 1.0465306425903456\n","05/03/2022 09:24:02 - INFO - __main__ -     global_step = 5103\n","05/03/2022 09:24:02 - INFO - __main__ -     loss = 0.08132013521523493\n","05/03/2022 09:24:02 - INFO - __main__ -   Total acc: 1.2980273663476156\n","05/03/2022 09:24:02 - INFO - __main__ -   Best Total acc: 1.3466883019126379\n","Epoch:  90% 9/10 [27:12<03:00, 180.52s/it]05/03/2022 09:24:02 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 09:24:02 - INFO - __main__ -   Loss: 0.4482046663761139\n","05/03/2022 09:24:02 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 09:24:02 - INFO - __main__ -   Loss: 0.5430426448583603\n","05/03/2022 09:26:50 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:26:50 - INFO - __main__ -     eval_accuracy = 0.6039146690125358\n","05/03/2022 09:26:50 - INFO - __main__ -     eval_loss = 1.5869856001334577\n","05/03/2022 09:26:50 - INFO - __main__ -     global_step = 5670\n","05/03/2022 09:26:50 - INFO - __main__ -     loss = 0.24441987784608962\n","05/03/2022 09:27:02 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:27:02 - INFO - __main__ -     eval_accuracy = 0.6962588473205258\n","05/03/2022 09:27:02 - INFO - __main__ -     eval_loss = 1.0626837648030052\n","05/03/2022 09:27:02 - INFO - __main__ -     global_step = 5670\n","05/03/2022 09:27:02 - INFO - __main__ -     loss = 0.06685519497677342\n","05/03/2022 09:27:02 - INFO - __main__ -   Total acc: 1.3001735163330617\n","05/03/2022 09:27:02 - INFO - __main__ -   Best Total acc: 1.3466883019126379\n","Epoch: 100% 10/10 [30:12<00:00, 181.20s/it]\n","05/03/2022 09:27:13 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:27:13 - INFO - __main__ -     eval_accuracy = 0.6039146690125358\n","05/03/2022 09:27:13 - INFO - __main__ -     eval_loss = 1.5869856001334577\n","05/03/2022 09:27:13 - INFO - __main__ -     global_step = 5670\n","05/03/2022 09:27:13 - INFO - __main__ -     loss = 0.24441987784608962\n","05/03/2022 09:27:24 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 09:27:24 - INFO - __main__ -     eval_accuracy = 0.6962588473205258\n","05/03/2022 09:27:24 - INFO - __main__ -     eval_loss = 1.0626837648030052\n","05/03/2022 09:27:24 - INFO - __main__ -     global_step = 5670\n","05/03/2022 09:27:24 - INFO - __main__ -     loss = 0.06685519497677342\n","05/03/2022 09:27:24 - INFO - __main__ -   Total acc: 1.3001735163330617\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/MTL_goemotion_orig_ekman_sqrt \\\n","  --tasks goemotion_taxonomy \\\n","  --sample 'sqrt'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"markdown","metadata":{"id":"x21BykKe8Rn1"},"source":["# Twitter sentiment and Goemotions in MTL settings"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2076131,"status":"ok","timestamp":1651575495164,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"iLXtmOR80dY0","outputId":"edf4c489-1713-44a5-8c25-fbd214e7d293"},"outputs":[{"name":"stdout","output_type":"stream","text":["05/03/2022 10:23:40 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 10:23:53 - INFO - __main__ -     Num Tasks = 2\n","05/03/2022 10:24:00 - INFO - __main__ -   ***** training data for goemotion_original *****\n","05/03/2022 10:24:00 - INFO - __main__ -     Data size = 36307\n","05/03/2022 10:29:07 - INFO - __main__ -   ***** training data for twitter_sentiment *****\n","05/03/2022 10:29:07 - INFO - __main__ -     Data size = 1599999\n","05/03/2022 10:29:13 - INFO - __main__ -     Num param = 113144982\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 10:29:14 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 10:29:14 - INFO - __main__ -   Loss: 3.7439968585968018\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 10:29:14 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 10:29:14 - INFO - __main__ -   Loss: 0.36936354637145996\n","05/03/2022 10:32:06 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:32:06 - INFO - __main__ -     eval_accuracy = 0.561469100505828\n","05/03/2022 10:32:06 - INFO - __main__ -     eval_loss = 1.656317466015556\n","05/03/2022 10:32:06 - INFO - __main__ -     global_step = 567\n","05/03/2022 10:32:06 - INFO - __main__ -     loss = 1.549111112715706\n","05/03/2022 10:32:07 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:32:07 - INFO - __main__ -     eval_accuracy = 0.505586592178771\n","05/03/2022 10:32:07 - INFO - __main__ -     eval_loss = 1.1950000395377478\n","05/03/2022 10:32:07 - INFO - __main__ -     global_step = 567\n","05/03/2022 10:32:07 - INFO - __main__ -     loss = 0.2031791687169403\n","05/03/2022 10:32:07 - INFO - __main__ -   Total acc: 1.0670556926845989\n","05/03/2022 10:32:08 - INFO - __main__ -   Best Total acc: 1.0670556926845989\n","Epoch:  10% 1/10 [02:54<26:11, 174.57s/it]05/03/2022 10:32:08 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 10:32:08 - INFO - __main__ -   Loss: 1.304236650466919\n","05/03/2022 10:32:09 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 10:32:09 - INFO - __main__ -   Loss: 0.2481418401002884\n","05/03/2022 10:35:00 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:35:00 - INFO - __main__ -     eval_accuracy = 0.5951176599956015\n","05/03/2022 10:35:00 - INFO - __main__ -     eval_loss = 1.3860000974788398\n","05/03/2022 10:35:00 - INFO - __main__ -     global_step = 1134\n","05/03/2022 10:35:00 - INFO - __main__ -     loss = 1.0246151366141318\n","05/03/2022 10:35:00 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:35:00 - INFO - __main__ -     eval_accuracy = 0.49162011173184356\n","05/03/2022 10:35:00 - INFO - __main__ -     eval_loss = 1.0495220227373971\n","05/03/2022 10:35:00 - INFO - __main__ -     global_step = 1134\n","05/03/2022 10:35:00 - INFO - __main__ -     loss = 0.14580895915969128\n","05/03/2022 10:35:00 - INFO - __main__ -   Total acc: 1.0867377717274451\n","05/03/2022 10:35:02 - INFO - __main__ -   Best Total acc: 1.0867377717274451\n","Epoch:  20% 2/10 [05:48<23:13, 174.14s/it]05/03/2022 10:35:02 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 10:35:02 - INFO - __main__ -   Loss: 0.5991067290306091\n","05/03/2022 10:35:02 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 10:35:02 - INFO - __main__ -   Loss: 0.6208295822143555\n","05/03/2022 10:37:53 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:37:53 - INFO - __main__ -     eval_accuracy = 0.6113921266769299\n","05/03/2022 10:37:53 - INFO - __main__ -     eval_loss = 1.3186831215825356\n","05/03/2022 10:37:53 - INFO - __main__ -     global_step = 1701\n","05/03/2022 10:37:53 - INFO - __main__ -     loss = 0.8106854303593779\n","05/03/2022 10:37:54 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:37:54 - INFO - __main__ -     eval_accuracy = 0.48044692737430167\n","05/03/2022 10:37:54 - INFO - __main__ -     eval_loss = 1.2704421079821058\n","05/03/2022 10:37:54 - INFO - __main__ -     global_step = 1701\n","05/03/2022 10:37:54 - INFO - __main__ -     loss = 0.16635190533897865\n","05/03/2022 10:37:54 - INFO - __main__ -   Total acc: 1.0918390540512315\n","05/03/2022 10:37:56 - INFO - __main__ -   Best Total acc: 1.0918390540512315\n","Epoch:  30% 3/10 [08:42<20:17, 173.98s/it]05/03/2022 10:37:56 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 10:37:56 - INFO - __main__ -   Loss: 1.1935486793518066\n","05/03/2022 10:37:56 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 10:37:56 - INFO - __main__ -   Loss: 1.384907066822052\n","05/03/2022 10:40:47 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:40:47 - INFO - __main__ -     eval_accuracy = 0.6267868924565647\n","05/03/2022 10:40:47 - INFO - __main__ -     eval_loss = 1.2707323443229672\n","05/03/2022 10:40:47 - INFO - __main__ -     global_step = 2268\n","05/03/2022 10:40:47 - INFO - __main__ -     loss = 0.7505030607840792\n","05/03/2022 10:40:48 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:40:48 - INFO - __main__ -     eval_accuracy = 0.49441340782122906\n","05/03/2022 10:40:48 - INFO - __main__ -     eval_loss = 1.310004119740592\n","05/03/2022 10:40:48 - INFO - __main__ -     global_step = 2268\n","05/03/2022 10:40:48 - INFO - __main__ -     loss = 0.16204320737924524\n","05/03/2022 10:40:48 - INFO - __main__ -   Total acc: 1.1212003002777937\n","05/03/2022 10:40:49 - INFO - __main__ -   Best Total acc: 1.1212003002777937\n","Epoch:  40% 4/10 [11:35<17:22, 173.82s/it]05/03/2022 10:40:49 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 10:40:49 - INFO - __main__ -   Loss: 1.0882501602172852\n","05/03/2022 10:40:50 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 10:40:50 - INFO - __main__ -   Loss: 1.24640554189682\n","05/03/2022 10:43:41 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:43:41 - INFO - __main__ -     eval_accuracy = 0.6239278645260612\n","05/03/2022 10:43:41 - INFO - __main__ -     eval_loss = 1.2794559144910902\n","05/03/2022 10:43:41 - INFO - __main__ -     global_step = 2835\n","05/03/2022 10:43:41 - INFO - __main__ -     loss = 0.6491807385411843\n","05/03/2022 10:43:42 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:43:42 - INFO - __main__ -     eval_accuracy = 0.48044692737430167\n","05/03/2022 10:43:42 - INFO - __main__ -     eval_loss = 1.3473196622398165\n","05/03/2022 10:43:42 - INFO - __main__ -     global_step = 2835\n","05/03/2022 10:43:42 - INFO - __main__ -     loss = 0.1515498884035376\n","05/03/2022 10:43:42 - INFO - __main__ -   Total acc: 1.1043747919003628\n","05/03/2022 10:43:42 - INFO - __main__ -   Best Total acc: 1.1212003002777937\n","Epoch:  50% 5/10 [14:28<14:26, 173.31s/it]05/03/2022 10:43:42 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 10:43:42 - INFO - __main__ -   Loss: 0.6673488020896912\n","05/03/2022 10:43:42 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 10:43:42 - INFO - __main__ -   Loss: 0.5573371350765228\n","05/03/2022 10:46:33 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:46:33 - INFO - __main__ -     eval_accuracy = 0.6239278645260612\n","05/03/2022 10:46:33 - INFO - __main__ -     eval_loss = 1.274055702248441\n","05/03/2022 10:46:33 - INFO - __main__ -     global_step = 3402\n","05/03/2022 10:46:33 - INFO - __main__ -     loss = 0.5554862991632398\n","05/03/2022 10:46:33 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:46:33 - INFO - __main__ -     eval_accuracy = 0.4860335195530726\n","05/03/2022 10:46:33 - INFO - __main__ -     eval_loss = 1.2552545828951729\n","05/03/2022 10:46:33 - INFO - __main__ -     global_step = 3402\n","05/03/2022 10:46:33 - INFO - __main__ -     loss = 0.16477820413873728\n","05/03/2022 10:46:33 - INFO - __main__ -   Total acc: 1.1099613840791338\n","05/03/2022 10:46:33 - INFO - __main__ -   Best Total acc: 1.1212003002777937\n","Epoch:  60% 6/10 [17:19<11:31, 172.77s/it]05/03/2022 10:46:34 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 10:46:34 - INFO - __main__ -   Loss: 0.47869783639907837\n","05/03/2022 10:46:34 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 10:46:34 - INFO - __main__ -   Loss: 0.467987984418869\n","05/03/2022 10:49:24 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:49:24 - INFO - __main__ -     eval_accuracy = 0.6223883879480977\n","05/03/2022 10:49:24 - INFO - __main__ -     eval_loss = 1.2703883119961918\n","05/03/2022 10:49:24 - INFO - __main__ -     global_step = 3969\n","05/03/2022 10:49:24 - INFO - __main__ -     loss = 0.5434710515344374\n","05/03/2022 10:49:25 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:49:25 - INFO - __main__ -     eval_accuracy = 0.49441340782122906\n","05/03/2022 10:49:25 - INFO - __main__ -     eval_loss = 1.2137770861387254\n","05/03/2022 10:49:25 - INFO - __main__ -     global_step = 3969\n","05/03/2022 10:49:25 - INFO - __main__ -     loss = 0.1562624803400965\n","05/03/2022 10:49:25 - INFO - __main__ -   Total acc: 1.1168017957693268\n","05/03/2022 10:49:25 - INFO - __main__ -   Best Total acc: 1.1212003002777937\n","Epoch:  70% 7/10 [20:11<08:37, 172.42s/it]05/03/2022 10:49:25 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 10:49:25 - INFO - __main__ -   Loss: 0.6862752437591553\n","05/03/2022 10:49:26 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 10:49:26 - INFO - __main__ -   Loss: 0.21307772397994995\n","05/03/2022 10:52:16 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:52:16 - INFO - __main__ -     eval_accuracy = 0.6212887618209809\n","05/03/2022 10:52:16 - INFO - __main__ -     eval_loss = 1.3028993354635625\n","05/03/2022 10:52:16 - INFO - __main__ -     global_step = 4536\n","05/03/2022 10:52:16 - INFO - __main__ -     loss = 0.45750798009060045\n","05/03/2022 10:52:17 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:52:17 - INFO - __main__ -     eval_accuracy = 0.4888268156424581\n","05/03/2022 10:52:17 - INFO - __main__ -     eval_loss = 1.2692280405097538\n","05/03/2022 10:52:17 - INFO - __main__ -     global_step = 4536\n","05/03/2022 10:52:17 - INFO - __main__ -     loss = 0.15646149055142042\n","05/03/2022 10:52:17 - INFO - __main__ -   Total acc: 1.110115577463439\n","05/03/2022 10:52:17 - INFO - __main__ -   Best Total acc: 1.1212003002777937\n","Epoch:  80% 8/10 [23:03<05:44, 172.26s/it]05/03/2022 10:52:17 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 10:52:17 - INFO - __main__ -   Loss: 0.4037903845310211\n","05/03/2022 10:52:17 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 10:52:17 - INFO - __main__ -   Loss: 0.1752227395772934\n","05/03/2022 10:55:08 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:55:08 - INFO - __main__ -     eval_accuracy = 0.6245876402023313\n","05/03/2022 10:55:08 - INFO - __main__ -     eval_loss = 1.3209929796089606\n","05/03/2022 10:55:08 - INFO - __main__ -     global_step = 5103\n","05/03/2022 10:55:08 - INFO - __main__ -     loss = 0.37569538683908116\n","05/03/2022 10:55:09 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:55:09 - INFO - __main__ -     eval_accuracy = 0.4888268156424581\n","05/03/2022 10:55:09 - INFO - __main__ -     eval_loss = 1.2965351230568356\n","05/03/2022 10:55:09 - INFO - __main__ -     global_step = 5103\n","05/03/2022 10:55:09 - INFO - __main__ -     loss = 0.16444811219681296\n","05/03/2022 10:55:09 - INFO - __main__ -   Total acc: 1.1134144558447894\n","05/03/2022 10:55:09 - INFO - __main__ -   Best Total acc: 1.1212003002777937\n","Epoch:  90% 9/10 [25:55<02:52, 172.15s/it]05/03/2022 10:55:09 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 10:55:09 - INFO - __main__ -   Loss: 0.6501252055168152\n","05/03/2022 10:55:09 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 10:55:09 - INFO - __main__ -   Loss: 0.6315462589263916\n","\n","05/03/2022 10:58:00 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:58:00 - INFO - __main__ -     eval_accuracy = 0.6274466681328348\n","05/03/2022 10:58:00 - INFO - __main__ -     eval_loss = 1.3211459626946174\n","05/03/2022 10:58:00 - INFO - __main__ -     global_step = 5670\n","05/03/2022 10:58:00 - INFO - __main__ -     loss = 0.35688291813330675\n","05/03/2022 10:58:01 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:58:01 - INFO - __main__ -     eval_accuracy = 0.4860335195530726\n","05/03/2022 10:58:01 - INFO - __main__ -     eval_loss = 1.2689815796083874\n","05/03/2022 10:58:01 - INFO - __main__ -     global_step = 5670\n","05/03/2022 10:58:01 - INFO - __main__ -     loss = 0.1672801306008031\n","05/03/2022 10:58:01 - INFO - __main__ -   Total acc: 1.1134801876859073\n","05/03/2022 10:58:01 - INFO - __main__ -   Best Total acc: 1.1212003002777937\n","Epoch: 100% 10/10 [28:47<00:00, 172.71s/it]\n","05/03/2022 10:58:11 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:58:11 - INFO - __main__ -     eval_accuracy = 0.6274466681328348\n","05/03/2022 10:58:11 - INFO - __main__ -     eval_loss = 1.3211459626946174\n","05/03/2022 10:58:11 - INFO - __main__ -     global_step = 5670\n","05/03/2022 10:58:11 - INFO - __main__ -     loss = 0.35688291813330675\n","05/03/2022 10:58:12 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 10:58:12 - INFO - __main__ -     eval_accuracy = 0.4860335195530726\n","05/03/2022 10:58:12 - INFO - __main__ -     eval_loss = 1.2689815796083874\n","05/03/2022 10:58:12 - INFO - __main__ -     global_step = 5670\n","05/03/2022 10:58:12 - INFO - __main__ -     loss = 0.1672801306008031\n","05/03/2022 10:58:12 - INFO - __main__ -   Total acc: 1.1134801876859073\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/MTL_goemotion_twitter \\\n","  --tasks goemotion_twitter_sentiment \\\n","  --sample 'anneal'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2058775,"status":"ok","timestamp":1651577553933,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"uHzL3b2I8p8n","outputId":"b1c10f22-48fb-49db-8a20-dbf3197e28d1"},"outputs":[{"name":"stdout","output_type":"stream","text":["05/03/2022 10:58:16 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 10:58:29 - INFO - __main__ -     Num Tasks = 2\n","05/03/2022 10:58:35 - INFO - __main__ -   ***** training data for goemotion_original *****\n","05/03/2022 10:58:35 - INFO - __main__ -     Data size = 36307\n","05/03/2022 11:03:42 - INFO - __main__ -   ***** training data for twitter_sentiment *****\n","05/03/2022 11:03:42 - INFO - __main__ -     Data size = 1599999\n","05/03/2022 11:03:48 - INFO - __main__ -     Num param = 113144982\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 11:03:48 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:03:48 - INFO - __main__ -   Loss: 3.7439968585968018\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 11:03:48 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:03:48 - INFO - __main__ -   Loss: 0.36936354637145996\n","05/03/2022 11:06:38 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:06:38 - INFO - __main__ -     eval_accuracy = 0.561469100505828\n","05/03/2022 11:06:38 - INFO - __main__ -     eval_loss = 1.656317466015556\n","05/03/2022 11:06:38 - INFO - __main__ -     global_step = 567\n","05/03/2022 11:06:38 - INFO - __main__ -     loss = 1.549111112715706\n","05/03/2022 11:06:39 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:06:39 - INFO - __main__ -     eval_accuracy = 0.505586592178771\n","05/03/2022 11:06:39 - INFO - __main__ -     eval_loss = 1.1950000395377478\n","05/03/2022 11:06:39 - INFO - __main__ -     global_step = 567\n","05/03/2022 11:06:39 - INFO - __main__ -     loss = 0.2031791687169403\n","05/03/2022 11:06:39 - INFO - __main__ -   Total acc: 1.0670556926845989\n","05/03/2022 11:06:40 - INFO - __main__ -   Best Total acc: 1.0670556926845989\n","Epoch:  10% 1/10 [02:52<25:52, 172.48s/it]05/03/2022 11:06:41 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:06:41 - INFO - __main__ -   Loss: 1.304236650466919\n","05/03/2022 11:06:41 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:06:41 - INFO - __main__ -   Loss: 0.2481418401002884\n","05/03/2022 11:09:30 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:09:30 - INFO - __main__ -     eval_accuracy = 0.6001759401803387\n","05/03/2022 11:09:30 - INFO - __main__ -     eval_loss = 1.3843044407128868\n","05/03/2022 11:09:30 - INFO - __main__ -     global_step = 1134\n","05/03/2022 11:09:30 - INFO - __main__ -     loss = 1.0392374675950886\n","05/03/2022 11:09:31 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:09:31 - INFO - __main__ -     eval_accuracy = 0.5027932960893855\n","05/03/2022 11:09:31 - INFO - __main__ -     eval_loss = 1.0368236091401841\n","05/03/2022 11:09:31 - INFO - __main__ -     global_step = 1134\n","05/03/2022 11:09:31 - INFO - __main__ -     loss = 0.14230118342387824\n","05/03/2022 11:09:31 - INFO - __main__ -   Total acc: 1.102969236269724\n","05/03/2022 11:09:33 - INFO - __main__ -   Best Total acc: 1.102969236269724\n","Epoch:  20% 2/10 [05:44<22:58, 172.28s/it]05/03/2022 11:09:33 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 11:09:33 - INFO - __main__ -   Loss: 0.5079278945922852\n","05/03/2022 11:09:33 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:09:33 - INFO - __main__ -   Loss: 0.7398175597190857\n","05/03/2022 11:12:22 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:12:22 - INFO - __main__ -     eval_accuracy = 0.6133714537057401\n","05/03/2022 11:12:22 - INFO - __main__ -     eval_loss = 1.3044900480913153\n","05/03/2022 11:12:22 - INFO - __main__ -     global_step = 1701\n","05/03/2022 11:12:22 - INFO - __main__ -     loss = 0.843078388423516\n","05/03/2022 11:12:23 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:12:23 - INFO - __main__ -     eval_accuracy = 0.4972067039106145\n","05/03/2022 11:12:23 - INFO - __main__ -     eval_loss = 1.304487845963902\n","05/03/2022 11:12:23 - INFO - __main__ -     global_step = 1701\n","05/03/2022 11:12:23 - INFO - __main__ -     loss = 0.1577479433288019\n","05/03/2022 11:12:23 - INFO - __main__ -   Total acc: 1.1105781576163545\n","05/03/2022 11:12:24 - INFO - __main__ -   Best Total acc: 1.1105781576163545\n","Epoch:  30% 3/10 [08:36<20:04, 172.05s/it]05/03/2022 11:12:25 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:12:25 - INFO - __main__ -   Loss: 1.8311222791671753\n","05/03/2022 11:12:25 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:12:25 - INFO - __main__ -   Loss: 1.4020390510559082\n","05/03/2022 11:15:14 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:15:14 - INFO - __main__ -     eval_accuracy = 0.6221684627226743\n","05/03/2022 11:15:14 - INFO - __main__ -     eval_loss = 1.2846701035966772\n","05/03/2022 11:15:14 - INFO - __main__ -     global_step = 2268\n","05/03/2022 11:15:14 - INFO - __main__ -     loss = 0.7919119342084074\n","05/03/2022 11:15:15 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:15:15 - INFO - __main__ -     eval_accuracy = 0.4888268156424581\n","05/03/2022 11:15:15 - INFO - __main__ -     eval_loss = 1.11743483973874\n","05/03/2022 11:15:15 - INFO - __main__ -     global_step = 2268\n","05/03/2022 11:15:15 - INFO - __main__ -     loss = 0.1465921491331945\n","05/03/2022 11:15:15 - INFO - __main__ -   Total acc: 1.1109952783651325\n","05/03/2022 11:15:16 - INFO - __main__ -   Best Total acc: 1.1109952783651325\n","Epoch:  40% 4/10 [11:28<17:11, 171.97s/it]05/03/2022 11:15:16 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:15:16 - INFO - __main__ -   Loss: 0.9186297655105591\n","05/03/2022 11:15:17 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:15:17 - INFO - __main__ -   Loss: 1.0587214231491089\n","05/03/2022 11:18:06 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:18:06 - INFO - __main__ -     eval_accuracy = 0.6149109302837036\n","05/03/2022 11:18:06 - INFO - __main__ -     eval_loss = 1.3008072397201886\n","05/03/2022 11:18:06 - INFO - __main__ -     global_step = 2835\n","05/03/2022 11:18:06 - INFO - __main__ -     loss = 0.701701374491266\n","05/03/2022 11:18:07 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:18:07 - INFO - __main__ -     eval_accuracy = 0.48324022346368717\n","05/03/2022 11:18:07 - INFO - __main__ -     eval_loss = 1.3181833075152503\n","05/03/2022 11:18:07 - INFO - __main__ -     global_step = 2835\n","05/03/2022 11:18:07 - INFO - __main__ -     loss = 0.1293268555896833\n","05/03/2022 11:18:07 - INFO - __main__ -   Total acc: 1.0981511537473907\n","05/03/2022 11:18:07 - INFO - __main__ -   Best Total acc: 1.1109952783651325\n","Epoch:  50% 5/10 [14:18<14:17, 171.45s/it]05/03/2022 11:18:07 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 11:18:07 - INFO - __main__ -   Loss: 0.4701278805732727\n","05/03/2022 11:18:07 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:18:07 - INFO - __main__ -   Loss: 0.4229213297367096\n","05/03/2022 11:20:56 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:20:56 - INFO - __main__ -     eval_accuracy = 0.6252474158786012\n","05/03/2022 11:20:56 - INFO - __main__ -     eval_loss = 1.2956647290234407\n","05/03/2022 11:20:56 - INFO - __main__ -     global_step = 3402\n","05/03/2022 11:20:56 - INFO - __main__ -     loss = 0.6021917458144025\n","05/03/2022 11:20:57 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:20:57 - INFO - __main__ -     eval_accuracy = 0.4776536312849162\n","05/03/2022 11:20:57 - INFO - __main__ -     eval_loss = 1.2567138347360822\n","05/03/2022 11:20:57 - INFO - __main__ -     global_step = 3402\n","05/03/2022 11:20:57 - INFO - __main__ -     loss = 0.14094413517328797\n","05/03/2022 11:20:57 - INFO - __main__ -   Total acc: 1.1029010471635174\n","05/03/2022 11:20:57 - INFO - __main__ -   Best Total acc: 1.1109952783651325\n","Epoch:  60% 6/10 [17:09<11:24, 171.04s/it]05/03/2022 11:20:57 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 11:20:57 - INFO - __main__ -   Loss: 0.5041301250457764\n","05/03/2022 11:20:57 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:20:57 - INFO - __main__ -   Loss: 0.5110483169555664\n","05/03/2022 11:23:46 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:23:46 - INFO - __main__ -     eval_accuracy = 0.6223883879480977\n","05/03/2022 11:23:46 - INFO - __main__ -     eval_loss = 1.2952159023819154\n","05/03/2022 11:23:46 - INFO - __main__ -     global_step = 3969\n","05/03/2022 11:23:46 - INFO - __main__ -     loss = 0.5951715647333303\n","05/03/2022 11:23:47 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:23:47 - INFO - __main__ -     eval_accuracy = 0.4888268156424581\n","05/03/2022 11:23:47 - INFO - __main__ -     eval_loss = 1.2524055116706425\n","05/03/2022 11:23:47 - INFO - __main__ -     global_step = 3969\n","05/03/2022 11:23:47 - INFO - __main__ -     loss = 0.13192893654055485\n","05/03/2022 11:23:47 - INFO - __main__ -   Total acc: 1.1112152035905558\n","05/03/2022 11:23:48 - INFO - __main__ -   Best Total acc: 1.1112152035905558\n","Epoch:  70% 7/10 [20:00<08:33, 171.15s/it]05/03/2022 11:23:49 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:23:49 - INFO - __main__ -   Loss: 0.7843151688575745\n","05/03/2022 11:23:49 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:23:49 - INFO - __main__ -   Loss: 0.1582426279783249\n","05/03/2022 11:26:38 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:26:38 - INFO - __main__ -     eval_accuracy = 0.618869584341324\n","05/03/2022 11:26:38 - INFO - __main__ -     eval_loss = 1.3344078186002473\n","05/03/2022 11:26:38 - INFO - __main__ -     global_step = 4536\n","05/03/2022 11:26:38 - INFO - __main__ -     loss = 0.49149987127835576\n","05/03/2022 11:26:39 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:26:39 - INFO - __main__ -     eval_accuracy = 0.49162011173184356\n","05/03/2022 11:26:39 - INFO - __main__ -     eval_loss = 1.2508371750513712\n","05/03/2022 11:26:39 - INFO - __main__ -     global_step = 4536\n","05/03/2022 11:26:39 - INFO - __main__ -     loss = 0.11301557950977716\n","05/03/2022 11:26:39 - INFO - __main__ -   Total acc: 1.1104896960731676\n","05/03/2022 11:26:39 - INFO - __main__ -   Best Total acc: 1.1112152035905558\n","Epoch:  80% 8/10 [22:50<05:41, 170.97s/it]05/03/2022 11:26:39 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:26:39 - INFO - __main__ -   Loss: 0.793945848941803\n","05/03/2022 11:26:39 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:26:39 - INFO - __main__ -   Loss: 0.7399069964885712\n","05/03/2022 11:29:28 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:29:28 - INFO - __main__ -     eval_accuracy = 0.6164504068616671\n","05/03/2022 11:29:28 - INFO - __main__ -     eval_loss = 1.3480069693079733\n","05/03/2022 11:29:28 - INFO - __main__ -     global_step = 5103\n","05/03/2022 11:29:28 - INFO - __main__ -     loss = 0.4155508854296683\n","05/03/2022 11:29:29 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:29:29 - INFO - __main__ -     eval_accuracy = 0.4860335195530726\n","05/03/2022 11:29:29 - INFO - __main__ -     eval_loss = 1.2855236490567525\n","05/03/2022 11:29:29 - INFO - __main__ -     global_step = 5103\n","05/03/2022 11:29:29 - INFO - __main__ -     loss = 0.12766734467297003\n","05/03/2022 11:29:29 - INFO - __main__ -   Total acc: 1.1024839264147397\n","05/03/2022 11:29:29 - INFO - __main__ -   Best Total acc: 1.1112152035905558\n","Epoch:  90% 9/10 [25:41<02:50, 170.76s/it]05/03/2022 11:29:29 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:29:29 - INFO - __main__ -   Loss: 0.7627900242805481\n","05/03/2022 11:29:30 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:29:30 - INFO - __main__ -   Loss: 0.7844162285327911\n","05/03/2022 11:32:18 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:32:18 - INFO - __main__ -     eval_accuracy = 0.6195293600175941\n","05/03/2022 11:32:18 - INFO - __main__ -     eval_loss = 1.345737329947718\n","05/03/2022 11:32:18 - INFO - __main__ -     global_step = 5670\n","05/03/2022 11:32:18 - INFO - __main__ -     loss = 0.42426101550643824\n","05/03/2022 11:32:19 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:32:19 - INFO - __main__ -     eval_accuracy = 0.48044692737430167\n","05/03/2022 11:32:19 - INFO - __main__ -     eval_loss = 1.225431928038597\n","05/03/2022 11:32:19 - INFO - __main__ -     global_step = 5670\n","05/03/2022 11:32:19 - INFO - __main__ -     loss = 0.1261751009463423\n","05/03/2022 11:32:19 - INFO - __main__ -   Total acc: 1.0999762873918957\n","05/03/2022 11:32:19 - INFO - __main__ -   Best Total acc: 1.1112152035905558\n","Epoch: 100% 10/10 [28:31<00:00, 171.14s/it]\n","05/03/2022 11:32:30 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:32:30 - INFO - __main__ -     eval_accuracy = 0.6195293600175941\n","05/03/2022 11:32:30 - INFO - __main__ -     eval_loss = 1.345737329947718\n","05/03/2022 11:32:30 - INFO - __main__ -     global_step = 5670\n","05/03/2022 11:32:30 - INFO - __main__ -     loss = 0.42426101550643824\n","05/03/2022 11:32:31 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:32:31 - INFO - __main__ -     eval_accuracy = 0.48044692737430167\n","05/03/2022 11:32:31 - INFO - __main__ -     eval_loss = 1.225431928038597\n","05/03/2022 11:32:31 - INFO - __main__ -     global_step = 5670\n","05/03/2022 11:32:31 - INFO - __main__ -     loss = 0.1261751009463423\n","05/03/2022 11:32:31 - INFO - __main__ -   Total acc: 1.0999762873918957\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/MTL_goemotion_twitter_prop \\\n","  --tasks goemotion_twitter_sentiment \\\n","  --sample 'prop'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2055067,"status":"ok","timestamp":1651579608975,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"9SBI3qvF8rVe","outputId":"19ac4288-fbf7-471a-950a-e5e5250eba35"},"outputs":[{"name":"stdout","output_type":"stream","text":["05/03/2022 11:32:34 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 11:32:47 - INFO - __main__ -     Num Tasks = 2\n","05/03/2022 11:32:53 - INFO - __main__ -   ***** training data for goemotion_original *****\n","05/03/2022 11:32:53 - INFO - __main__ -     Data size = 36307\n","05/03/2022 11:37:55 - INFO - __main__ -   ***** training data for twitter_sentiment *****\n","05/03/2022 11:37:55 - INFO - __main__ -     Data size = 1599999\n","05/03/2022 11:38:01 - INFO - __main__ -     Num param = 113144982\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 11:38:01 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:38:01 - INFO - __main__ -   Loss: 3.7439968585968018\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 11:38:02 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:38:02 - INFO - __main__ -   Loss: 0.36936354637145996\n","05/03/2022 11:40:51 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:40:51 - INFO - __main__ -     eval_accuracy = 0.5359577743567188\n","05/03/2022 11:40:51 - INFO - __main__ -     eval_loss = 1.7422809272428388\n","05/03/2022 11:40:51 - INFO - __main__ -     global_step = 567\n","05/03/2022 11:40:51 - INFO - __main__ -     loss = 1.3740701870943504\n","05/03/2022 11:40:52 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:40:52 - INFO - __main__ -     eval_accuracy = 0.5167597765363129\n","05/03/2022 11:40:52 - INFO - __main__ -     eval_loss = 1.2430862877104016\n","05/03/2022 11:40:52 - INFO - __main__ -     global_step = 567\n","05/03/2022 11:40:52 - INFO - __main__ -     loss = 0.24615661305830383\n","05/03/2022 11:40:52 - INFO - __main__ -   Total acc: 1.0527175508930315\n","05/03/2022 11:40:54 - INFO - __main__ -   Best Total acc: 1.0527175508930315\n","Epoch:  10% 1/10 [02:52<25:50, 172.32s/it]05/03/2022 11:40:54 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:40:54 - INFO - __main__ -   Loss: 1.6964174509048462\n","05/03/2022 11:40:54 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:40:54 - INFO - __main__ -   Loss: 0.28025293350219727\n","05/03/2022 11:43:43 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:43:43 - INFO - __main__ -     eval_accuracy = 0.5977567627006818\n","05/03/2022 11:43:43 - INFO - __main__ -     eval_loss = 1.4018142502525779\n","05/03/2022 11:43:43 - INFO - __main__ -     global_step = 1134\n","05/03/2022 11:43:43 - INFO - __main__ -     loss = 0.9450022009524929\n","05/03/2022 11:43:44 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:43:44 - INFO - __main__ -     eval_accuracy = 0.49441340782122906\n","05/03/2022 11:43:44 - INFO - __main__ -     eval_loss = 1.1866920772526\n","05/03/2022 11:43:44 - INFO - __main__ -     global_step = 1134\n","05/03/2022 11:43:44 - INFO - __main__ -     loss = 0.17338792606531206\n","05/03/2022 11:43:44 - INFO - __main__ -   Total acc: 1.092170170521911\n","05/03/2022 11:43:45 - INFO - __main__ -   Best Total acc: 1.092170170521911\n","Epoch:  20% 2/10 [05:44<22:56, 172.07s/it]05/03/2022 11:43:46 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 11:43:46 - INFO - __main__ -   Loss: 0.3970349133014679\n","05/03/2022 11:43:46 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:43:46 - INFO - __main__ -   Loss: 0.7076511979103088\n","05/03/2022 11:46:35 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:46:35 - INFO - __main__ -     eval_accuracy = 0.6186496591159006\n","05/03/2022 11:46:35 - INFO - __main__ -     eval_loss = 1.3208055668011283\n","05/03/2022 11:46:35 - INFO - __main__ -     global_step = 1701\n","05/03/2022 11:46:35 - INFO - __main__ -     loss = 0.7490458811191445\n","05/03/2022 11:46:36 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:46:36 - INFO - __main__ -     eval_accuracy = 0.48324022346368717\n","05/03/2022 11:46:36 - INFO - __main__ -     eval_loss = 1.2375976582368216\n","05/03/2022 11:46:36 - INFO - __main__ -     global_step = 1701\n","05/03/2022 11:46:36 - INFO - __main__ -     loss = 0.18539580382157045\n","05/03/2022 11:46:36 - INFO - __main__ -   Total acc: 1.1018898825795878\n","05/03/2022 11:46:37 - INFO - __main__ -   Best Total acc: 1.1018898825795878\n","Epoch:  30% 3/10 [08:36<20:03, 171.98s/it]05/03/2022 11:46:38 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:46:38 - INFO - __main__ -   Loss: 1.307035207748413\n","05/03/2022 11:46:38 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:46:38 - INFO - __main__ -   Loss: 1.1838268041610718\n","05/03/2022 11:49:27 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:49:27 - INFO - __main__ -     eval_accuracy = 0.6223883879480977\n","05/03/2022 11:49:27 - INFO - __main__ -     eval_loss = 1.268406407380565\n","05/03/2022 11:49:27 - INFO - __main__ -     global_step = 2268\n","05/03/2022 11:49:27 - INFO - __main__ -     loss = 0.7288636187817474\n","05/03/2022 11:49:28 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:49:28 - INFO - __main__ -     eval_accuracy = 0.48324022346368717\n","05/03/2022 11:49:28 - INFO - __main__ -     eval_loss = 1.2186729927857718\n","05/03/2022 11:49:28 - INFO - __main__ -     global_step = 2268\n","05/03/2022 11:49:28 - INFO - __main__ -     loss = 0.17580464936963675\n","05/03/2022 11:49:28 - INFO - __main__ -   Total acc: 1.1056286114117848\n","05/03/2022 11:49:29 - INFO - __main__ -   Best Total acc: 1.1056286114117848\n","Epoch:  40% 4/10 [11:27<17:11, 171.88s/it]05/03/2022 11:49:29 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:49:29 - INFO - __main__ -   Loss: 1.223541498184204\n","05/03/2022 11:49:30 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:49:30 - INFO - __main__ -   Loss: 1.1365807056427002\n","05/03/2022 11:52:19 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:52:19 - INFO - __main__ -     eval_accuracy = 0.6250274906531779\n","05/03/2022 11:52:19 - INFO - __main__ -     eval_loss = 1.2771633834641931\n","05/03/2022 11:52:19 - INFO - __main__ -     global_step = 2835\n","05/03/2022 11:52:19 - INFO - __main__ -     loss = 0.6659050841915964\n","05/03/2022 11:52:20 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:52:20 - INFO - __main__ -     eval_accuracy = 0.4776536312849162\n","05/03/2022 11:52:20 - INFO - __main__ -     eval_loss = 1.344697544972102\n","05/03/2022 11:52:20 - INFO - __main__ -     global_step = 2835\n","05/03/2022 11:52:20 - INFO - __main__ -     loss = 0.1556076916489862\n","05/03/2022 11:52:20 - INFO - __main__ -   Total acc: 1.102681121938094\n","05/03/2022 11:52:20 - INFO - __main__ -   Best Total acc: 1.1056286114117848\n","Epoch:  50% 5/10 [14:18<14:17, 171.41s/it]05/03/2022 11:52:20 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 11:52:20 - INFO - __main__ -   Loss: 0.309747576713562\n","05/03/2022 11:52:20 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:52:20 - INFO - __main__ -   Loss: 0.350647509098053\n","05/03/2022 11:55:09 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:55:09 - INFO - __main__ -     eval_accuracy = 0.6241477897514845\n","05/03/2022 11:55:09 - INFO - __main__ -     eval_loss = 1.2804708866589098\n","05/03/2022 11:55:09 - INFO - __main__ -     global_step = 3402\n","05/03/2022 11:55:09 - INFO - __main__ -     loss = 0.5495712461715443\n","05/03/2022 11:55:10 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:55:10 - INFO - __main__ -     eval_accuracy = 0.49441340782122906\n","05/03/2022 11:55:10 - INFO - __main__ -     eval_loss = 1.2427202979723613\n","05/03/2022 11:55:10 - INFO - __main__ -     global_step = 3402\n","05/03/2022 11:55:10 - INFO - __main__ -     loss = 0.16400188183973705\n","05/03/2022 11:55:10 - INFO - __main__ -   Total acc: 1.1185611975727137\n","05/03/2022 11:55:12 - INFO - __main__ -   Best Total acc: 1.1185611975727137\n","Epoch:  60% 6/10 [17:10<11:26, 171.67s/it]05/03/2022 11:55:12 - INFO - __main__ -   Task: 1, Step: 0\n","05/03/2022 11:55:12 - INFO - __main__ -   Loss: 0.47806036472320557\n","05/03/2022 11:55:12 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 11:55:12 - INFO - __main__ -   Loss: 0.4583820104598999\n","05/03/2022 11:58:01 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:58:01 - INFO - __main__ -     eval_accuracy = 0.6276665933582581\n","05/03/2022 11:58:01 - INFO - __main__ -     eval_loss = 1.2626239951497014\n","05/03/2022 11:58:01 - INFO - __main__ -     global_step = 3969\n","05/03/2022 11:58:01 - INFO - __main__ -     loss = 0.5551675766233414\n","05/03/2022 11:58:02 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 11:58:02 - INFO - __main__ -     eval_accuracy = 0.4776536312849162\n","05/03/2022 11:58:02 - INFO - __main__ -     eval_loss = 1.2389507833454343\n","05/03/2022 11:58:02 - INFO - __main__ -     global_step = 3969\n","05/03/2022 11:58:02 - INFO - __main__ -     loss = 0.15354995006396446\n","05/03/2022 11:58:02 - INFO - __main__ -   Total acc: 1.1053202246431744\n","05/03/2022 11:58:02 - INFO - __main__ -   Best Total acc: 1.1185611975727137\n","Epoch:  70% 7/10 [20:00<08:33, 171.26s/it]05/03/2022 11:58:02 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 11:58:02 - INFO - __main__ -   Loss: 0.8522693514823914\n","05/03/2022 11:58:03 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 11:58:03 - INFO - __main__ -   Loss: 0.11831744015216827\n","05/03/2022 12:00:51 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:00:51 - INFO - __main__ -     eval_accuracy = 0.6243677149769079\n","05/03/2022 12:00:51 - INFO - __main__ -     eval_loss = 1.2840399740031399\n","05/03/2022 12:00:51 - INFO - __main__ -     global_step = 4536\n","05/03/2022 12:00:51 - INFO - __main__ -     loss = 0.5281210388245826\n","05/03/2022 12:00:52 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:00:52 - INFO - __main__ -     eval_accuracy = 0.49441340782122906\n","05/03/2022 12:00:52 - INFO - __main__ -     eval_loss = 1.2418056592345237\n","05/03/2022 12:00:52 - INFO - __main__ -     global_step = 4536\n","05/03/2022 12:00:52 - INFO - __main__ -     loss = 0.15131050981646912\n","05/03/2022 12:00:52 - INFO - __main__ -   Total acc: 1.118781122798137\n","05/03/2022 12:00:54 - INFO - __main__ -   Best Total acc: 1.118781122798137\n","Epoch:  80% 8/10 [22:52<05:42, 171.26s/it]05/03/2022 12:00:54 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:00:54 - INFO - __main__ -   Loss: 0.9010326266288757\n","05/03/2022 12:00:54 - INFO - __main__ -   Task: 1, Step: 1\n","05/03/2022 12:00:54 - INFO - __main__ -   Loss: 0.10432617366313934\n","05/03/2022 12:03:43 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:03:43 - INFO - __main__ -     eval_accuracy = 0.6287662194853749\n","05/03/2022 12:03:43 - INFO - __main__ -     eval_loss = 1.2955406271903294\n","05/03/2022 12:03:43 - INFO - __main__ -     global_step = 5103\n","05/03/2022 12:03:43 - INFO - __main__ -     loss = 0.4243138868499685\n","05/03/2022 12:03:44 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:03:44 - INFO - __main__ -     eval_accuracy = 0.4776536312849162\n","05/03/2022 12:03:44 - INFO - __main__ -     eval_loss = 1.2570119437244203\n","05/03/2022 12:03:44 - INFO - __main__ -     global_step = 5103\n","05/03/2022 12:03:44 - INFO - __main__ -     loss = 0.1495243415262754\n","05/03/2022 12:03:44 - INFO - __main__ -   Total acc: 1.1064198507702911\n","05/03/2022 12:03:44 - INFO - __main__ -   Best Total acc: 1.118781122798137\n","Epoch:  90% 9/10 [25:42<02:51, 171.06s/it]05/03/2022 12:03:44 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:03:44 - INFO - __main__ -   Loss: 0.7532606720924377\n","05/03/2022 12:03:45 - INFO - __main__ -   Task: 0, Step: 1\n","05/03/2022 12:03:45 - INFO - __main__ -   Loss: 0.6466403603553772\n","05/03/2022 12:06:33 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:06:33 - INFO - __main__ -     eval_accuracy = 0.6239278645260612\n","05/03/2022 12:06:33 - INFO - __main__ -     eval_loss = 1.3080978262351142\n","05/03/2022 12:06:33 - INFO - __main__ -     global_step = 5670\n","05/03/2022 12:06:33 - INFO - __main__ -     loss = 0.3914295808663444\n","05/03/2022 12:06:34 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:06:34 - INFO - __main__ -     eval_accuracy = 0.4888268156424581\n","05/03/2022 12:06:34 - INFO - __main__ -     eval_loss = 1.2569675243563123\n","05/03/2022 12:06:34 - INFO - __main__ -     global_step = 5670\n","05/03/2022 12:06:34 - INFO - __main__ -     loss = 0.1524665990098654\n","05/03/2022 12:06:34 - INFO - __main__ -   Total acc: 1.1127546801685193\n","05/03/2022 12:06:34 - INFO - __main__ -   Best Total acc: 1.118781122798137\n","Epoch: 100% 10/10 [28:32<00:00, 171.29s/it]\n","05/03/2022 12:06:45 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:06:45 - INFO - __main__ -     eval_accuracy = 0.6239278645260612\n","05/03/2022 12:06:45 - INFO - __main__ -     eval_loss = 1.3080978262351142\n","05/03/2022 12:06:45 - INFO - __main__ -     global_step = 5670\n","05/03/2022 12:06:45 - INFO - __main__ -     loss = 0.3914295808663444\n","05/03/2022 12:06:46 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:06:46 - INFO - __main__ -     eval_accuracy = 0.4888268156424581\n","05/03/2022 12:06:46 - INFO - __main__ -     eval_loss = 1.2569675243563123\n","05/03/2022 12:06:46 - INFO - __main__ -     global_step = 5670\n","05/03/2022 12:06:46 - INFO - __main__ -     loss = 0.1524665990098654\n","05/03/2022 12:06:46 - INFO - __main__ -   Total acc: 1.1127546801685193\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/MTL_goemotion_twitter_sqrt \\\n","  --tasks goemotion_twitter_sentiment \\\n","  --sample 'sqrt'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"UdD1MvSi-t_7"},"outputs":[],"source":[""]},{"cell_type":"markdown","metadata":{"id":"3XgSeBPp-uTu"},"source":["**SINGLE TASK**"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1745665,"status":"ok","timestamp":1651581521829,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"zLqi4g6e9UT7","outputId":"c802d0c2-3504-4f80-cc47-5a032b6d183d"},"outputs":[{"name":"stdout","output_type":"stream","text":["05/03/2022 12:09:37 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 12:09:43 - INFO - __main__ -     Num Tasks = 1\n","05/03/2022 12:09:50 - INFO - __main__ -   ***** training data for goemotion_ekman *****\n","05/03/2022 12:09:50 - INFO - __main__ -     Data size = 39554\n","05/03/2022 12:09:50 - INFO - __main__ -     Num param = 111307459\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 12:09:50 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:09:50 - INFO - __main__ -   Loss: 1.9757914543151855\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 12:12:43 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:12:43 - INFO - __main__ -     eval_accuracy = 0.6837209302325581\n","05/03/2022 12:12:43 - INFO - __main__ -     eval_loss = 0.8570063755349504\n","05/03/2022 12:12:43 - INFO - __main__ -     global_step = 618\n","05/03/2022 12:12:43 - INFO - __main__ -     loss = 1.0961532436046013\n","05/03/2022 12:12:43 - INFO - __main__ -   Total acc: 0.6837209302325581\n","05/03/2022 12:12:44 - INFO - __main__ -   Best Total acc: 0.6837209302325581\n","Epoch:  10% 1/10 [02:54<26:08, 174.26s/it]05/03/2022 12:12:45 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:12:45 - INFO - __main__ -   Loss: 1.044395089149475\n","05/03/2022 12:15:36 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:15:36 - INFO - __main__ -     eval_accuracy = 0.7104145601617796\n","05/03/2022 12:15:36 - INFO - __main__ -     eval_loss = 0.7711958028075191\n","05/03/2022 12:15:36 - INFO - __main__ -     global_step = 1236\n","05/03/2022 12:15:36 - INFO - __main__ -     loss = 0.7994471969245707\n","05/03/2022 12:15:36 - INFO - __main__ -   Total acc: 0.7104145601617796\n","05/03/2022 12:15:38 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  20% 2/10 [05:47<23:09, 173.72s/it]05/03/2022 12:15:38 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:15:38 - INFO - __main__ -   Loss: 1.4250825643539429\n","05/03/2022 12:18:29 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:18:29 - INFO - __main__ -     eval_accuracy = 0.7089989888776542\n","05/03/2022 12:18:29 - INFO - __main__ -     eval_loss = 0.800586885572539\n","05/03/2022 12:18:29 - INFO - __main__ -     global_step = 1854\n","05/03/2022 12:18:29 - INFO - __main__ -     loss = 0.6708436091837374\n","05/03/2022 12:18:29 - INFO - __main__ -   Total acc: 0.7089989888776542\n","05/03/2022 12:18:29 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  30% 3/10 [08:39<20:09, 172.75s/it]05/03/2022 12:18:30 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:18:30 - INFO - __main__ -   Loss: 0.49933454394340515\n","05/03/2022 12:21:21 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:21:21 - INFO - __main__ -     eval_accuracy = 0.7049544994944388\n","05/03/2022 12:21:21 - INFO - __main__ -     eval_loss = 0.806046071244172\n","05/03/2022 12:21:21 - INFO - __main__ -     global_step = 2472\n","05/03/2022 12:21:21 - INFO - __main__ -     loss = 0.5412542391749262\n","05/03/2022 12:21:21 - INFO - __main__ -   Total acc: 0.7049544994944388\n","05/03/2022 12:21:21 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  40% 4/10 [11:30<17:13, 172.25s/it]05/03/2022 12:21:21 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:21:21 - INFO - __main__ -   Loss: 0.3924572467803955\n","05/03/2022 12:24:12 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:24:12 - INFO - __main__ -     eval_accuracy = 0.6948432760364004\n","05/03/2022 12:24:12 - INFO - __main__ -     eval_loss = 0.9843737292667679\n","05/03/2022 12:24:12 - INFO - __main__ -     global_step = 3090\n","05/03/2022 12:24:12 - INFO - __main__ -     loss = 0.40357939442047025\n","05/03/2022 12:24:12 - INFO - __main__ -   Total acc: 0.6948432760364004\n","05/03/2022 12:24:12 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  50% 5/10 [14:21<14:19, 171.89s/it]05/03/2022 12:24:12 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:24:12 - INFO - __main__ -   Loss: 0.15502244234085083\n","05/03/2022 12:27:03 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:27:03 - INFO - __main__ -     eval_accuracy = 0.6942366026289181\n","05/03/2022 12:27:03 - INFO - __main__ -     eval_loss = 1.0139791354716052\n","05/03/2022 12:27:03 - INFO - __main__ -     global_step = 3708\n","05/03/2022 12:27:03 - INFO - __main__ -     loss = 0.24803267633934237\n","05/03/2022 12:27:03 - INFO - __main__ -   Total acc: 0.6942366026289181\n","05/03/2022 12:27:03 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  60% 6/10 [17:13<11:26, 171.71s/it]05/03/2022 12:27:04 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:27:04 - INFO - __main__ -   Loss: 0.14888370037078857\n","05/03/2022 12:29:55 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:29:55 - INFO - __main__ -     eval_accuracy = 0.6962588473205258\n","05/03/2022 12:29:55 - INFO - __main__ -     eval_loss = 1.2157431197975594\n","05/03/2022 12:29:55 - INFO - __main__ -     global_step = 4326\n","05/03/2022 12:29:55 - INFO - __main__ -     loss = 0.15793381747412932\n","05/03/2022 12:29:55 - INFO - __main__ -   Total acc: 0.6962588473205258\n","05/03/2022 12:29:55 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  70% 7/10 [20:04<08:34, 171.57s/it]05/03/2022 12:29:55 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:29:55 - INFO - __main__ -   Loss: 0.08469051867723465\n","05/03/2022 12:32:46 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:32:46 - INFO - __main__ -     eval_accuracy = 0.6936299292214358\n","05/03/2022 12:32:46 - INFO - __main__ -     eval_loss = 1.31224556825378\n","05/03/2022 12:32:46 - INFO - __main__ -     global_step = 4944\n","05/03/2022 12:32:46 - INFO - __main__ -     loss = 0.0844125410023486\n","05/03/2022 12:32:46 - INFO - __main__ -   Total acc: 0.6936299292214358\n","05/03/2022 12:32:46 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  80% 8/10 [22:55<05:43, 171.51s/it]05/03/2022 12:32:46 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:32:46 - INFO - __main__ -   Loss: 0.017433078959584236\n","05/03/2022 12:35:37 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:35:37 - INFO - __main__ -     eval_accuracy = 0.6946410515672397\n","05/03/2022 12:35:37 - INFO - __main__ -     eval_loss = 1.3691813784095899\n","05/03/2022 12:35:37 - INFO - __main__ -     global_step = 5562\n","05/03/2022 12:35:37 - INFO - __main__ -     loss = 0.053646754933834605\n","05/03/2022 12:35:37 - INFO - __main__ -   Total acc: 0.6946410515672397\n","05/03/2022 12:35:37 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch:  90% 9/10 [25:47<02:51, 171.46s/it]05/03/2022 12:35:38 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:35:38 - INFO - __main__ -   Loss: 0.017333246767520905\n","05/03/2022 12:38:29 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:38:29 - INFO - __main__ -     eval_accuracy = 0.6944388270980789\n","05/03/2022 12:38:29 - INFO - __main__ -     eval_loss = 1.387793476778904\n","05/03/2022 12:38:29 - INFO - __main__ -     global_step = 6180\n","05/03/2022 12:38:29 - INFO - __main__ -     loss = 0.03456100012465082\n","05/03/2022 12:38:29 - INFO - __main__ -   Total acc: 0.6944388270980789\n","05/03/2022 12:38:29 - INFO - __main__ -   Best Total acc: 0.7104145601617796\n","Epoch: 100% 10/10 [28:38<00:00, 171.87s/it]\n","05/03/2022 12:38:41 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 12:38:41 - INFO - __main__ -     eval_accuracy = 0.6944388270980789\n","05/03/2022 12:38:41 - INFO - __main__ -     eval_loss = 1.387793476778904\n","05/03/2022 12:38:41 - INFO - __main__ -     global_step = 6180\n","05/03/2022 12:38:41 - INFO - __main__ -     loss = 0.03456100012465082\n","05/03/2022 12:38:41 - INFO - __main__ -   Total acc: 0.6944388270980789\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/goemotion_ekman \\\n","  --tasks single \\\n","  --task_id 2 \\\n","  --sample 'rr'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wPg6v7cb9Ue2","outputId":"12c28948-9d99-4dfd-b25d-f361b55364a0"},"outputs":[{"output_type":"stream","name":"stdout","text":["05/03/2022 12:38:43 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","05/03/2022 12:38:55 - INFO - __main__ -     Num Tasks = 1\n","05/03/2022 12:44:00 - INFO - __main__ -   ***** training data for twitter_sentiment *****\n","05/03/2022 12:44:00 - INFO - __main__ -     Data size = 1599999\n","05/03/2022 12:44:06 - INFO - __main__ -     Num param = 111303614\n","Epoch:   0% 0/10 [00:00<?, ?it/s]05/03/2022 12:44:06 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 12:44:06 - INFO - __main__ -   Loss: 0.6775206923484802\n","/content/drive/MyDrive/769/CS769_Project/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1055.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","05/03/2022 12:48:29 - INFO - __main__ -   Task: 0, Step: 1000\n","05/03/2022 12:48:29 - INFO - __main__ -   Loss: 0.623634269812724\n","05/03/2022 12:52:53 - INFO - __main__ -   Task: 0, Step: 2000\n","05/03/2022 12:52:53 - INFO - __main__ -   Loss: 0.5335792278659636\n","05/03/2022 12:57:15 - INFO - __main__ -   Task: 0, Step: 3000\n","05/03/2022 12:57:15 - INFO - __main__ -   Loss: 0.4934201787925092\n","05/03/2022 13:01:39 - INFO - __main__ -   Task: 0, Step: 4000\n","05/03/2022 13:01:39 - INFO - __main__ -   Loss: 0.4682154101756059\n","05/03/2022 13:06:02 - INFO - __main__ -   Task: 0, Step: 5000\n","05/03/2022 13:06:02 - INFO - __main__ -   Loss: 0.4515676970089514\n","05/03/2022 13:10:25 - INFO - __main__ -   Task: 0, Step: 6000\n","05/03/2022 13:10:25 - INFO - __main__ -   Loss: 0.4399985406120069\n","05/03/2022 13:14:48 - INFO - __main__ -   Task: 0, Step: 7000\n","05/03/2022 13:14:48 - INFO - __main__ -   Loss: 0.4298085942321668\n","05/03/2022 13:19:11 - INFO - __main__ -   Task: 0, Step: 8000\n","05/03/2022 13:19:11 - INFO - __main__ -   Loss: 0.42177396147061313\n","05/03/2022 13:23:34 - INFO - __main__ -   Task: 0, Step: 9000\n","05/03/2022 13:23:34 - INFO - __main__ -   Loss: 0.4143500138831965\n","05/03/2022 13:27:57 - INFO - __main__ -   Task: 0, Step: 10000\n","05/03/2022 13:27:57 - INFO - __main__ -   Loss: 0.4082040848479952\n","05/03/2022 13:32:20 - INFO - __main__ -   Task: 0, Step: 11000\n","05/03/2022 13:32:20 - INFO - __main__ -   Loss: 0.40236595687300236\n","05/03/2022 13:36:43 - INFO - __main__ -   Task: 0, Step: 12000\n","05/03/2022 13:36:43 - INFO - __main__ -   Loss: 0.39799106492878306\n","05/03/2022 13:41:06 - INFO - __main__ -   Task: 0, Step: 13000\n","05/03/2022 13:41:06 - INFO - __main__ -   Loss: 0.39405721176778724\n","05/03/2022 13:45:29 - INFO - __main__ -   Task: 0, Step: 14000\n","05/03/2022 13:45:29 - INFO - __main__ -   Loss: 0.39030288346045255\n","05/03/2022 13:49:52 - INFO - __main__ -   Task: 0, Step: 15000\n","05/03/2022 13:49:52 - INFO - __main__ -   Loss: 0.3867502451051268\n","05/03/2022 13:54:15 - INFO - __main__ -   Task: 0, Step: 16000\n","05/03/2022 13:54:15 - INFO - __main__ -   Loss: 0.383774751478244\n","05/03/2022 13:58:38 - INFO - __main__ -   Task: 0, Step: 17000\n","05/03/2022 13:58:38 - INFO - __main__ -   Loss: 0.38068884439250594\n","05/03/2022 14:03:00 - INFO - __main__ -   Task: 0, Step: 18000\n","05/03/2022 14:03:00 - INFO - __main__ -   Loss: 0.378234025382938\n","05/03/2022 14:07:23 - INFO - __main__ -   Task: 0, Step: 19000\n","05/03/2022 14:07:23 - INFO - __main__ -   Loss: 0.3757397310137686\n","05/03/2022 14:11:45 - INFO - __main__ -   Task: 0, Step: 20000\n","05/03/2022 14:11:45 - INFO - __main__ -   Loss: 0.37346602666691525\n","05/03/2022 14:16:08 - INFO - __main__ -   Task: 0, Step: 21000\n","05/03/2022 14:16:08 - INFO - __main__ -   Loss: 0.3714098393325323\n","05/03/2022 14:20:30 - INFO - __main__ -   Task: 0, Step: 22000\n","05/03/2022 14:20:30 - INFO - __main__ -   Loss: 0.3694391537194068\n","05/03/2022 14:24:53 - INFO - __main__ -   Task: 0, Step: 23000\n","05/03/2022 14:24:53 - INFO - __main__ -   Loss: 0.36796369342249446\n","05/03/2022 14:29:16 - INFO - __main__ -   Task: 0, Step: 24000\n","05/03/2022 14:29:16 - INFO - __main__ -   Loss: 0.36608019917594914\n","05/03/2022 14:33:39 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 14:33:39 - INFO - __main__ -     eval_accuracy = 0.49441340782122906\n","05/03/2022 14:33:39 - INFO - __main__ -     eval_loss = 1.515906652642621\n","05/03/2022 14:33:39 - INFO - __main__ -     global_step = 24999\n","05/03/2022 14:33:39 - INFO - __main__ -     loss = 0.36456127604967004\n","05/03/2022 14:33:39 - INFO - __main__ -   Total acc: 0.49441340782122906\n","05/03/2022 14:33:40 - INFO - __main__ -   Best Total acc: 0.49441340782122906\n","Epoch:  10% 1/10 [1:49:34<16:26:09, 6574.41s/it]05/03/2022 14:33:41 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 14:33:41 - INFO - __main__ -   Loss: 0.24744510650634766\n","05/03/2022 14:38:02 - INFO - __main__ -   Task: 0, Step: 1000\n","05/03/2022 14:38:02 - INFO - __main__ -   Loss: 0.3159816575261739\n","05/03/2022 14:42:23 - INFO - __main__ -   Task: 0, Step: 2000\n","05/03/2022 14:42:23 - INFO - __main__ -   Loss: 0.31629655970775267\n","05/03/2022 14:46:44 - INFO - __main__ -   Task: 0, Step: 3000\n","05/03/2022 14:46:44 - INFO - __main__ -   Loss: 0.31560982815159117\n","05/03/2022 14:51:05 - INFO - __main__ -   Task: 0, Step: 4000\n","05/03/2022 14:51:05 - INFO - __main__ -   Loss: 0.31712039495275723\n","05/03/2022 14:55:26 - INFO - __main__ -   Task: 0, Step: 5000\n","05/03/2022 14:55:26 - INFO - __main__ -   Loss: 0.3175535194458091\n","05/03/2022 14:59:48 - INFO - __main__ -   Task: 0, Step: 6000\n","05/03/2022 14:59:48 - INFO - __main__ -   Loss: 0.31748351070348907\n","05/03/2022 15:04:09 - INFO - __main__ -   Task: 0, Step: 7000\n","05/03/2022 15:04:09 - INFO - __main__ -   Loss: 0.3172563139523222\n","05/03/2022 15:08:30 - INFO - __main__ -   Task: 0, Step: 8000\n","05/03/2022 15:08:30 - INFO - __main__ -   Loss: 0.31663125012069476\n","05/03/2022 15:12:51 - INFO - __main__ -   Task: 0, Step: 9000\n","05/03/2022 15:12:51 - INFO - __main__ -   Loss: 0.31621379747369294\n","05/03/2022 15:17:12 - INFO - __main__ -   Task: 0, Step: 10000\n","05/03/2022 15:17:12 - INFO - __main__ -   Loss: 0.3160884967924011\n","05/03/2022 15:21:33 - INFO - __main__ -   Task: 0, Step: 11000\n","05/03/2022 15:21:33 - INFO - __main__ -   Loss: 0.3151114896155197\n","05/03/2022 15:25:54 - INFO - __main__ -   Task: 0, Step: 12000\n","05/03/2022 15:25:54 - INFO - __main__ -   Loss: 0.31433861668721186\n","05/03/2022 15:30:15 - INFO - __main__ -   Task: 0, Step: 13000\n","05/03/2022 15:30:15 - INFO - __main__ -   Loss: 0.31376751941581404\n","05/03/2022 15:34:36 - INFO - __main__ -   Task: 0, Step: 14000\n","05/03/2022 15:34:36 - INFO - __main__ -   Loss: 0.3137263525041901\n","05/03/2022 15:38:57 - INFO - __main__ -   Task: 0, Step: 15000\n","05/03/2022 15:38:57 - INFO - __main__ -   Loss: 0.31333062619429797\n","05/03/2022 15:43:17 - INFO - __main__ -   Task: 0, Step: 16000\n","05/03/2022 15:43:17 - INFO - __main__ -   Loss: 0.3127411652893493\n","05/03/2022 15:47:38 - INFO - __main__ -   Task: 0, Step: 17000\n","05/03/2022 15:47:38 - INFO - __main__ -   Loss: 0.3120202862997629\n","05/03/2022 15:51:59 - INFO - __main__ -   Task: 0, Step: 18000\n","05/03/2022 15:51:59 - INFO - __main__ -   Loss: 0.31160800631856145\n","05/03/2022 15:56:20 - INFO - __main__ -   Task: 0, Step: 19000\n","05/03/2022 15:56:20 - INFO - __main__ -   Loss: 0.31122792271615757\n","05/03/2022 16:00:41 - INFO - __main__ -   Task: 0, Step: 20000\n","05/03/2022 16:00:41 - INFO - __main__ -   Loss: 0.31074192220576374\n","05/03/2022 16:05:02 - INFO - __main__ -   Task: 0, Step: 21000\n","05/03/2022 16:05:02 - INFO - __main__ -   Loss: 0.3103562715579927\n","05/03/2022 16:09:23 - INFO - __main__ -   Task: 0, Step: 22000\n","05/03/2022 16:09:23 - INFO - __main__ -   Loss: 0.30996381625628605\n","05/03/2022 16:13:43 - INFO - __main__ -   Task: 0, Step: 23000\n","05/03/2022 16:13:43 - INFO - __main__ -   Loss: 0.30976060102536834\n","05/03/2022 16:18:04 - INFO - __main__ -   Task: 0, Step: 24000\n","05/03/2022 16:18:04 - INFO - __main__ -   Loss: 0.30955413599084886\n","05/03/2022 16:22:26 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 16:22:26 - INFO - __main__ -     eval_accuracy = 0.4972067039106145\n","05/03/2022 16:22:26 - INFO - __main__ -     eval_loss = 1.4316032038794624\n","05/03/2022 16:22:26 - INFO - __main__ -     global_step = 49998\n","05/03/2022 16:22:26 - INFO - __main__ -     loss = 0.30945506316095456\n","05/03/2022 16:22:26 - INFO - __main__ -   Total acc: 0.4972067039106145\n","05/03/2022 16:22:27 - INFO - __main__ -   Best Total acc: 0.4972067039106145\n","Epoch:  20% 2/10 [3:38:21<14:32:50, 6546.37s/it]05/03/2022 16:22:27 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 16:22:27 - INFO - __main__ -   Loss: 0.2117902636528015\n","05/03/2022 16:26:48 - INFO - __main__ -   Task: 0, Step: 1000\n","05/03/2022 16:26:48 - INFO - __main__ -   Loss: 0.30124660010461685\n","05/03/2022 16:31:09 - INFO - __main__ -   Task: 0, Step: 2000\n","05/03/2022 16:31:09 - INFO - __main__ -   Loss: 0.2995197141486248\n","05/03/2022 16:35:29 - INFO - __main__ -   Task: 0, Step: 3000\n","05/03/2022 16:35:29 - INFO - __main__ -   Loss: 0.2992130083027918\n","05/03/2022 16:39:49 - INFO - __main__ -   Task: 0, Step: 4000\n","05/03/2022 16:39:49 - INFO - __main__ -   Loss: 0.29746976542181597\n","05/03/2022 16:44:09 - INFO - __main__ -   Task: 0, Step: 5000\n","05/03/2022 16:44:09 - INFO - __main__ -   Loss: 0.2979927298640268\n","05/03/2022 16:48:30 - INFO - __main__ -   Task: 0, Step: 6000\n","05/03/2022 16:48:30 - INFO - __main__ -   Loss: 0.29790426206872317\n","05/03/2022 16:52:50 - INFO - __main__ -   Task: 0, Step: 7000\n","05/03/2022 16:52:50 - INFO - __main__ -   Loss: 0.2972517743969565\n","05/03/2022 16:57:10 - INFO - __main__ -   Task: 0, Step: 8000\n","05/03/2022 16:57:10 - INFO - __main__ -   Loss: 0.29674421498940046\n","05/03/2022 17:01:31 - INFO - __main__ -   Task: 0, Step: 9000\n","05/03/2022 17:01:31 - INFO - __main__ -   Loss: 0.2956984913066227\n","05/03/2022 17:05:52 - INFO - __main__ -   Task: 0, Step: 10000\n","05/03/2022 17:05:52 - INFO - __main__ -   Loss: 0.29468720435012985\n","05/03/2022 17:10:13 - INFO - __main__ -   Task: 0, Step: 11000\n","05/03/2022 17:10:13 - INFO - __main__ -   Loss: 0.29318774239741807\n","05/03/2022 17:14:34 - INFO - __main__ -   Task: 0, Step: 12000\n","05/03/2022 17:14:34 - INFO - __main__ -   Loss: 0.29268251084341934\n","05/03/2022 17:18:55 - INFO - __main__ -   Task: 0, Step: 13000\n","05/03/2022 17:18:55 - INFO - __main__ -   Loss: 0.29196452176183035\n","05/03/2022 17:23:17 - INFO - __main__ -   Task: 0, Step: 14000\n","05/03/2022 17:23:17 - INFO - __main__ -   Loss: 0.29108926307381533\n","05/03/2022 17:27:38 - INFO - __main__ -   Task: 0, Step: 15000\n","05/03/2022 17:27:38 - INFO - __main__ -   Loss: 0.2898680585445277\n","05/03/2022 17:32:00 - INFO - __main__ -   Task: 0, Step: 16000\n","05/03/2022 17:32:00 - INFO - __main__ -   Loss: 0.2891002570899805\n","05/03/2022 17:36:23 - INFO - __main__ -   Task: 0, Step: 17000\n","05/03/2022 17:36:23 - INFO - __main__ -   Loss: 0.28769240660278367\n","05/03/2022 17:40:44 - INFO - __main__ -   Task: 0, Step: 18000\n","05/03/2022 17:40:44 - INFO - __main__ -   Loss: 0.286763840579674\n","05/03/2022 17:45:06 - INFO - __main__ -   Task: 0, Step: 19000\n","05/03/2022 17:45:06 - INFO - __main__ -   Loss: 0.28558411725929245\n","05/03/2022 17:49:27 - INFO - __main__ -   Task: 0, Step: 20000\n","05/03/2022 17:49:27 - INFO - __main__ -   Loss: 0.28439612478568377\n","05/03/2022 17:53:48 - INFO - __main__ -   Task: 0, Step: 21000\n","05/03/2022 17:53:48 - INFO - __main__ -   Loss: 0.28322000609879594\n","05/03/2022 17:58:09 - INFO - __main__ -   Task: 0, Step: 22000\n","05/03/2022 17:58:09 - INFO - __main__ -   Loss: 0.2819091031169272\n","05/03/2022 18:02:30 - INFO - __main__ -   Task: 0, Step: 23000\n","05/03/2022 18:02:30 - INFO - __main__ -   Loss: 0.2810004269155576\n","05/03/2022 18:06:51 - INFO - __main__ -   Task: 0, Step: 24000\n","05/03/2022 18:06:51 - INFO - __main__ -   Loss: 0.2794808306327378\n","05/03/2022 18:11:13 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 18:11:13 - INFO - __main__ -     eval_accuracy = 0.4888268156424581\n","05/03/2022 18:11:13 - INFO - __main__ -     eval_loss = 1.8012464864386453\n","05/03/2022 18:11:13 - INFO - __main__ -     global_step = 74997\n","05/03/2022 18:11:13 - INFO - __main__ -     loss = 0.2782474249649646\n","05/03/2022 18:11:13 - INFO - __main__ -   Total acc: 0.4888268156424581\n","05/03/2022 18:11:13 - INFO - __main__ -   Best Total acc: 0.4972067039106145\n","Epoch:  30% 3/10 [5:27:07<12:42:39, 6537.05s/it]05/03/2022 18:11:13 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 18:11:13 - INFO - __main__ -   Loss: 0.4557151198387146\n","05/03/2022 18:15:34 - INFO - __main__ -   Task: 0, Step: 1000\n","05/03/2022 18:15:34 - INFO - __main__ -   Loss: 0.2281243380430814\n","05/03/2022 18:19:55 - INFO - __main__ -   Task: 0, Step: 2000\n","05/03/2022 18:19:55 - INFO - __main__ -   Loss: 0.2273288983879597\n","05/03/2022 18:24:16 - INFO - __main__ -   Task: 0, Step: 3000\n","05/03/2022 18:24:16 - INFO - __main__ -   Loss: 0.2258360605204018\n","05/03/2022 18:28:37 - INFO - __main__ -   Task: 0, Step: 4000\n","05/03/2022 18:28:37 - INFO - __main__ -   Loss: 0.22545654191203757\n","05/03/2022 18:32:57 - INFO - __main__ -   Task: 0, Step: 5000\n","05/03/2022 18:32:57 - INFO - __main__ -   Loss: 0.22396320626892\n","05/03/2022 18:37:18 - INFO - __main__ -   Task: 0, Step: 6000\n","05/03/2022 18:37:18 - INFO - __main__ -   Loss: 0.22200539483198511\n","05/03/2022 18:41:38 - INFO - __main__ -   Task: 0, Step: 7000\n","05/03/2022 18:41:38 - INFO - __main__ -   Loss: 0.2207402709558651\n","05/03/2022 18:45:59 - INFO - __main__ -   Task: 0, Step: 8000\n","05/03/2022 18:45:59 - INFO - __main__ -   Loss: 0.21930707008910208\n","05/03/2022 18:50:20 - INFO - __main__ -   Task: 0, Step: 9000\n","05/03/2022 18:50:20 - INFO - __main__ -   Loss: 0.21812601836369067\n","05/03/2022 18:54:41 - INFO - __main__ -   Task: 0, Step: 10000\n","05/03/2022 18:54:41 - INFO - __main__ -   Loss: 0.21716749496868243\n","05/03/2022 18:59:02 - INFO - __main__ -   Task: 0, Step: 11000\n","05/03/2022 18:59:02 - INFO - __main__ -   Loss: 0.21546259140833682\n","05/03/2022 19:03:22 - INFO - __main__ -   Task: 0, Step: 12000\n","05/03/2022 19:03:22 - INFO - __main__ -   Loss: 0.2139047288343853\n","05/03/2022 19:07:44 - INFO - __main__ -   Task: 0, Step: 13000\n","05/03/2022 19:07:44 - INFO - __main__ -   Loss: 0.21299328534128129\n","05/03/2022 19:12:09 - INFO - __main__ -   Task: 0, Step: 14000\n","05/03/2022 19:12:09 - INFO - __main__ -   Loss: 0.21244197654874505\n","05/03/2022 19:16:44 - INFO - __main__ -   Task: 0, Step: 15000\n","05/03/2022 19:16:44 - INFO - __main__ -   Loss: 0.21178975673456363\n","05/03/2022 19:21:18 - INFO - __main__ -   Task: 0, Step: 16000\n","05/03/2022 19:21:18 - INFO - __main__ -   Loss: 0.21071328222274344\n","05/03/2022 19:25:53 - INFO - __main__ -   Task: 0, Step: 17000\n","05/03/2022 19:25:53 - INFO - __main__ -   Loss: 0.20970468372831846\n","05/03/2022 19:30:20 - INFO - __main__ -   Task: 0, Step: 18000\n","05/03/2022 19:30:20 - INFO - __main__ -   Loss: 0.20889426855675963\n","05/03/2022 19:34:52 - INFO - __main__ -   Task: 0, Step: 19000\n","05/03/2022 19:34:52 - INFO - __main__ -   Loss: 0.2083261187109561\n","05/03/2022 19:39:27 - INFO - __main__ -   Task: 0, Step: 20000\n","05/03/2022 19:39:27 - INFO - __main__ -   Loss: 0.20761371616850205\n","05/03/2022 19:44:01 - INFO - __main__ -   Task: 0, Step: 21000\n","05/03/2022 19:44:01 - INFO - __main__ -   Loss: 0.20687394683120644\n","05/03/2022 19:48:36 - INFO - __main__ -   Task: 0, Step: 22000\n","05/03/2022 19:48:36 - INFO - __main__ -   Loss: 0.20627302027455482\n","05/03/2022 19:52:58 - INFO - __main__ -   Task: 0, Step: 23000\n","05/03/2022 19:52:58 - INFO - __main__ -   Loss: 0.20575468680227324\n","05/03/2022 19:57:19 - INFO - __main__ -   Task: 0, Step: 24000\n","05/03/2022 19:57:19 - INFO - __main__ -   Loss: 0.20533363228924328\n","05/03/2022 20:01:40 - INFO - __main__ -   ***** Eval results *****\n","05/03/2022 20:01:40 - INFO - __main__ -     eval_accuracy = 0.49162011173184356\n","05/03/2022 20:01:40 - INFO - __main__ -     eval_loss = 1.9223022043704987\n","05/03/2022 20:01:40 - INFO - __main__ -     global_step = 99996\n","05/03/2022 20:01:40 - INFO - __main__ -     loss = 0.20487881225144736\n","05/03/2022 20:01:40 - INFO - __main__ -   Total acc: 0.49162011173184356\n","05/03/2022 20:01:40 - INFO - __main__ -   Best Total acc: 0.4972067039106145\n","Epoch:  40% 4/10 [7:17:33<10:57:14, 6572.48s/it]05/03/2022 20:01:40 - INFO - __main__ -   Task: 0, Step: 0\n","05/03/2022 20:01:40 - INFO - __main__ -   Loss: 0.08941683173179626\n","05/03/2022 20:06:01 - INFO - __main__ -   Task: 0, Step: 1000\n","05/03/2022 20:06:01 - INFO - __main__ -   Loss: 0.1915099689705329\n","05/03/2022 20:10:22 - INFO - __main__ -   Task: 0, Step: 2000\n","05/03/2022 20:10:22 - INFO - __main__ -   Loss: 0.18816854302836084\n","05/03/2022 20:14:43 - INFO - __main__ -   Task: 0, Step: 3000\n","05/03/2022 20:14:43 - INFO - __main__ -   Loss: 0.18828751720917578\n","05/03/2022 20:19:04 - INFO - __main__ -   Task: 0, Step: 4000\n","05/03/2022 20:19:04 - INFO - __main__ -   Loss: 0.18654466325440905\n","05/03/2022 20:23:25 - INFO - __main__ -   Task: 0, Step: 5000\n","05/03/2022 20:23:25 - INFO - __main__ -   Loss: 0.18646015371092617\n","05/03/2022 20:27:46 - INFO - __main__ -   Task: 0, Step: 6000\n","05/03/2022 20:27:46 - INFO - __main__ -   Loss: 0.18575242199959696\n","05/03/2022 20:32:06 - INFO - __main__ -   Task: 0, Step: 7000\n","05/03/2022 20:32:06 - INFO - __main__ -   Loss: 0.18433831451247076\n","05/03/2022 20:36:27 - INFO - __main__ -   Task: 0, Step: 8000\n","05/03/2022 20:36:27 - INFO - __main__ -   Loss: 0.18408767881895985\n","05/03/2022 20:40:48 - INFO - __main__ -   Task: 0, Step: 9000\n","05/03/2022 20:40:48 - INFO - __main__ -   Loss: 0.18292501526144417\n","05/03/2022 20:45:09 - INFO - __main__ -   Task: 0, Step: 10000\n","05/03/2022 20:45:09 - INFO - __main__ -   Loss: 0.18202430866619576\n","05/03/2022 20:49:29 - INFO - __main__ -   Task: 0, Step: 11000\n","05/03/2022 20:49:29 - INFO - __main__ -   Loss: 0.18031711431523081\n","05/03/2022 20:53:50 - INFO - __main__ -   Task: 0, Step: 12000\n","05/03/2022 20:53:50 - INFO - __main__ -   Loss: 0.17962807615961746\n","05/03/2022 20:58:11 - INFO - __main__ -   Task: 0, Step: 13000\n","05/03/2022 20:58:11 - INFO - __main__ -   Loss: 0.17837850108018344\n","05/03/2022 21:02:32 - INFO - __main__ -   Task: 0, Step: 14000\n","05/03/2022 21:02:32 - INFO - __main__ -   Loss: 0.17728264171157332\n","05/03/2022 21:06:53 - INFO - __main__ -   Task: 0, Step: 15000\n","05/03/2022 21:06:53 - INFO - __main__ -   Loss: 0.175578916009855\n","05/03/2022 21:11:14 - INFO - __main__ -   Task: 0, Step: 16000\n","05/03/2022 21:11:14 - INFO - __main__ -   Loss: 0.17456748672373326\n","05/03/2022 21:15:35 - INFO - __main__ -   Task: 0, Step: 17000\n","05/03/2022 21:15:35 - INFO - __main__ -   Loss: 0.17276878147339178\n","05/03/2022 21:19:56 - INFO - __main__ -   Task: 0, Step: 18000\n","05/03/2022 21:19:56 - INFO - __main__ -   Loss: 0.17154065250040829\n","05/03/2022 21:24:17 - INFO - __main__ -   Task: 0, Step: 19000\n","05/03/2022 21:24:17 - INFO - __main__ -   Loss: 0.17013667331430404\n","05/03/2022 21:28:39 - INFO - __main__ -   Task: 0, Step: 20000\n","05/03/2022 21:28:39 - INFO - __main__ -   Loss: 0.1687572809874685\n","05/03/2022 21:33:03 - INFO - __main__ -   Task: 0, Step: 21000\n","05/03/2022 21:33:03 - INFO - __main__ -   Loss: 0.16717350943275935\n","05/03/2022 21:37:30 - INFO - __main__ -   Task: 0, Step: 22000\n","05/03/2022 21:37:30 - INFO - __main__ -   Loss: 0.1656362571480146\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir ./ckpt/twitter_sentiment \\\n","  --tasks single \\\n","  --task_id 3 \\\n","  --sample 'rr'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir ./data/ \\\n","  --vocab_file ./ckpt/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file ./config/MTL_pals_config.json \\\n","  --init_checkpoint ./ckpt/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"saRf2-qjJPso"},"outputs":[],"source":["# previous runs"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1956996,"status":"ok","timestamp":1649480914122,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"AT0d3N3pgnRF","outputId":"ae57003e-cd45-40df-e966-b495bd0480b7"},"outputs":[{"name":"stdout","output_type":"stream","text":["04/09/2022 04:35:58 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","04/09/2022 04:36:06 - INFO - __main__ -     Num Tasks = 1\n","04/09/2022 04:36:14 - INFO - __main__ -   ***** training data for goemotion *****\n","04/09/2022 04:36:14 - INFO - __main__ -     Data size = 36307\n","04/09/2022 04:36:14 - INFO - __main__ -     Num param = 111323608\n","Epoch:   0% 0/10 [00:00<?, ?it/s]04/09/2022 04:36:15 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:36:15 - INFO - __main__ -   Loss: 3.2078166007995605\n","/content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/optimization.py:154: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1050.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","04/09/2022 04:39:27 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 04:39:27 - INFO - __main__ -     eval_accuracy = 0.5830217725973169\n","04/09/2022 04:39:27 - INFO - __main__ -     eval_loss = 1.505824140012788\n","04/09/2022 04:39:27 - INFO - __main__ -     global_step = 567\n","04/09/2022 04:39:27 - INFO - __main__ -     loss = 2.126264854188108\n","04/09/2022 04:39:27 - INFO - __main__ -   Total acc: 0.5830217725973169\n","04/09/2022 04:39:29 - INFO - __main__ -   Best Total acc: 0.5830217725973169\n","Epoch:  10% 1/10 [03:14<29:09, 194.44s/it]04/09/2022 04:39:29 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:39:29 - INFO - __main__ -   Loss: 1.8263652324676514\n","04/09/2022 04:42:41 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 04:42:41 - INFO - __main__ -     eval_accuracy = 0.6173301077633605\n","04/09/2022 04:42:41 - INFO - __main__ -     eval_loss = 1.3121336842044167\n","04/09/2022 04:42:41 - INFO - __main__ -     global_step = 1134\n","04/09/2022 04:42:41 - INFO - __main__ -     loss = 1.387866456672628\n","04/09/2022 04:42:41 - INFO - __main__ -   Total acc: 0.6173301077633605\n","04/09/2022 04:42:43 - INFO - __main__ -   Best Total acc: 0.6173301077633605\n","Epoch:  20% 2/10 [06:28<25:54, 194.27s/it]04/09/2022 04:42:43 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:42:43 - INFO - __main__ -   Loss: 2.4894521236419678\n","04/09/2022 04:45:55 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 04:45:55 - INFO - __main__ -     eval_accuracy = 0.6278865185836815\n","04/09/2022 04:45:55 - INFO - __main__ -     eval_loss = 1.2719206687698046\n","04/09/2022 04:45:55 - INFO - __main__ -     global_step = 1701\n","04/09/2022 04:45:55 - INFO - __main__ -     loss = 1.1944080902890974\n","04/09/2022 04:45:55 - INFO - __main__ -   Total acc: 0.6278865185836815\n","04/09/2022 04:45:57 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch:  30% 3/10 [09:42<22:38, 194.04s/it]04/09/2022 04:45:57 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:45:57 - INFO - __main__ -   Loss: 1.0382184982299805\n","04/09/2022 04:49:09 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 04:49:09 - INFO - __main__ -     eval_accuracy = 0.6230481636243678\n","04/09/2022 04:49:09 - INFO - __main__ -     eval_loss = 1.2848746932139505\n","04/09/2022 04:49:09 - INFO - __main__ -     global_step = 2268\n","04/09/2022 04:49:09 - INFO - __main__ -     loss = 0.9547795444148769\n","04/09/2022 04:49:09 - INFO - __main__ -   Total acc: 0.6230481636243678\n","04/09/2022 04:49:09 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch:  40% 4/10 [12:54<19:19, 193.19s/it]04/09/2022 04:49:09 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:49:09 - INFO - __main__ -   Loss: 0.9361604452133179\n","04/09/2022 04:52:20 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 04:52:20 - INFO - __main__ -     eval_accuracy = 0.6234880140752145\n","04/09/2022 04:52:20 - INFO - __main__ -     eval_loss = 1.3793322820026552\n","04/09/2022 04:52:20 - INFO - __main__ -     global_step = 2835\n","04/09/2022 04:52:20 - INFO - __main__ -     loss = 0.8269730292825682\n","04/09/2022 04:52:20 - INFO - __main__ -   Total acc: 0.6234880140752145\n","04/09/2022 04:52:20 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch:  50% 5/10 [16:05<16:02, 192.55s/it]04/09/2022 04:52:20 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:52:20 - INFO - __main__ -   Loss: 1.0113962888717651\n","04/09/2022 04:55:31 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 04:55:31 - INFO - __main__ -     eval_accuracy = 0.6063338464921927\n","04/09/2022 04:55:31 - INFO - __main__ -     eval_loss = 1.4129166739791474\n","04/09/2022 04:55:31 - INFO - __main__ -     global_step = 3402\n","04/09/2022 04:55:31 - INFO - __main__ -     loss = 0.583207471398029\n","04/09/2022 04:55:31 - INFO - __main__ -   Total acc: 0.6063338464921927\n","04/09/2022 04:55:31 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch:  60% 6/10 [19:17<12:48, 192.19s/it]04/09/2022 04:55:32 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:55:32 - INFO - __main__ -   Loss: 0.8977932333946228\n","04/09/2022 04:58:43 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 04:58:43 - INFO - __main__ -     eval_accuracy = 0.6010556410820321\n","04/09/2022 04:58:43 - INFO - __main__ -     eval_loss = 1.5629901492048337\n","04/09/2022 04:58:43 - INFO - __main__ -     global_step = 3969\n","04/09/2022 04:58:43 - INFO - __main__ -     loss = 0.489638707322944\n","04/09/2022 04:58:43 - INFO - __main__ -   Total acc: 0.6010556410820321\n","04/09/2022 04:58:43 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch:  70% 7/10 [22:28<09:35, 191.96s/it]04/09/2022 04:58:43 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 04:58:43 - INFO - __main__ -   Loss: 0.3330917954444885\n","04/09/2022 05:01:55 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 05:01:55 - INFO - __main__ -     eval_accuracy = 0.5907191554871344\n","04/09/2022 05:01:55 - INFO - __main__ -     eval_loss = 1.598303008988264\n","04/09/2022 05:01:55 - INFO - __main__ -     global_step = 4536\n","04/09/2022 05:01:55 - INFO - __main__ -     loss = 0.3109843980987446\n","04/09/2022 05:01:55 - INFO - __main__ -   Total acc: 0.5907191554871344\n","04/09/2022 05:01:55 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch:  80% 8/10 [25:40<06:24, 192.04s/it]04/09/2022 05:01:55 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 05:01:55 - INFO - __main__ -   Loss: 0.23310768604278564\n","04/09/2022 05:05:07 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 05:05:07 - INFO - __main__ -     eval_accuracy = 0.5915988563888278\n","04/09/2022 05:05:07 - INFO - __main__ -     eval_loss = 1.7021918733897234\n","04/09/2022 05:05:07 - INFO - __main__ -     global_step = 5103\n","04/09/2022 05:05:07 - INFO - __main__ -     loss = 0.27654089127557197\n","04/09/2022 05:05:07 - INFO - __main__ -   Total acc: 0.5915988563888278\n","04/09/2022 05:05:07 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch:  90% 9/10 [28:53<03:12, 192.09s/it]04/09/2022 05:05:08 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 05:05:08 - INFO - __main__ -   Loss: 0.16524657607078552\n","04/09/2022 05:08:19 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 05:08:19 - INFO - __main__ -     eval_accuracy = 0.5931383329667913\n","04/09/2022 05:08:19 - INFO - __main__ -     eval_loss = 1.7077673678736485\n","04/09/2022 05:08:19 - INFO - __main__ -     global_step = 5670\n","04/09/2022 05:08:19 - INFO - __main__ -     loss = 0.19321389117884258\n","04/09/2022 05:08:19 - INFO - __main__ -   Total acc: 0.5931383329667913\n","04/09/2022 05:08:19 - INFO - __main__ -   Best Total acc: 0.6278865185836815\n","Epoch: 100% 10/10 [32:04<00:00, 192.47s/it]\n","04/09/2022 05:08:33 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 05:08:33 - INFO - __main__ -     eval_accuracy = 0.5931383329667913\n","04/09/2022 05:08:33 - INFO - __main__ -     eval_loss = 1.7077673678736485\n","04/09/2022 05:08:33 - INFO - __main__ -     global_step = 5670\n","04/09/2022 05:08:33 - INFO - __main__ -     loss = 0.19321389117884258\n","04/09/2022 05:08:33 - INFO - __main__ -   Total acc: 0.5931383329667913\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/Tmp_Model/goemotion \\\n","  --tasks single \\\n","  --task_id 1 \\\n","  --sample 'rr'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/data/ \\\n","  --vocab_file /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/config/pals_config.json \\\n","  --init_checkpoint /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1073622,"status":"ok","timestamp":1649493331228,"user":{"displayName":"ABHAY KUMAR","userId":"10336133233497407079"},"user_tz":300},"id":"Ykeicxr7_Foy","outputId":"fc8a32f2-8893-421a-8fec-caf40865795b"},"outputs":[{"name":"stdout","output_type":"stream","text":["04/09/2022 07:39:23 - INFO - __main__ -   device cuda n_gpu 1 distributed training False\n","04/09/2022 07:39:30 - INFO - __main__ -     Num Tasks = 1\n","04/09/2022 07:39:42 - INFO - __main__ -   ***** training data for sst *****\n","04/09/2022 07:39:42 - INFO - __main__ -     Data size = 67349\n","04/09/2022 07:39:42 - INFO - __main__ -     Num param = 111303614\n","Epoch:   0% 0/10 [00:00<?, ?it/s]04/09/2022 07:39:42 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 07:39:42 - INFO - __main__ -   Loss: 0.679380476474762\n","/content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/optimization.py:159: UserWarning: This overload of add_ is deprecated:\n","\tadd_(Number alpha, Tensor other)\n","Consider using one of the following signatures instead:\n","\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1050.)\n","  next_m.mul_(beta1).add_(1 - beta1, grad)\n","04/09/2022 07:45:00 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 07:45:00 - INFO - __main__ -   Loss: 0.2897664847989361\n","04/09/2022 07:45:18 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 07:45:18 - INFO - __main__ -     eval_accuracy = 0.9162844036697247\n","04/09/2022 07:45:18 - INFO - __main__ -     eval_loss = 0.21369404181701326\n","04/09/2022 07:45:18 - INFO - __main__ -     global_step = 1052\n","04/09/2022 07:45:18 - INFO - __main__ -     loss = 0.2866095604105877\n","04/09/2022 07:45:18 - INFO - __main__ -   Total acc: 0.9162844036697247\n","04/09/2022 07:45:20 - INFO - __main__ -   Best Total acc: 0.9162844036697247\n","Epoch:  10% 1/10 [05:37<50:41, 337.94s/it]04/09/2022 07:45:20 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 07:45:20 - INFO - __main__ -   Loss: 0.15050236880779266\n","04/09/2022 07:50:35 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 07:50:35 - INFO - __main__ -   Loss: 0.16125674454772568\n","04/09/2022 07:50:54 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 07:50:54 - INFO - __main__ -     eval_accuracy = 0.9185779816513762\n","04/09/2022 07:50:54 - INFO - __main__ -     eval_loss = 0.22706716736658997\n","04/09/2022 07:50:54 - INFO - __main__ -     global_step = 2104\n","04/09/2022 07:50:54 - INFO - __main__ -     loss = 0.16028587820766202\n","04/09/2022 07:50:54 - INFO - __main__ -   Total acc: 0.9185779816513762\n","04/09/2022 07:50:56 - INFO - __main__ -   Best Total acc: 0.9185779816513762\n","Epoch:  20% 2/10 [11:13<44:51, 336.49s/it]04/09/2022 07:50:56 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 07:50:56 - INFO - __main__ -   Loss: 0.13104064762592316\n","04/09/2022 07:56:10 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 07:56:10 - INFO - __main__ -   Loss: 0.1072656315524358\n","04/09/2022 07:56:29 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 07:56:29 - INFO - __main__ -     eval_accuracy = 0.9208715596330275\n","04/09/2022 07:56:29 - INFO - __main__ -     eval_loss = 0.21617308592663034\n","04/09/2022 07:56:29 - INFO - __main__ -     global_step = 3156\n","04/09/2022 07:56:29 - INFO - __main__ -     loss = 0.10715640216185836\n","04/09/2022 07:56:29 - INFO - __main__ -   Total acc: 0.9208715596330275\n","04/09/2022 07:56:30 - INFO - __main__ -   Best Total acc: 0.9208715596330275\n","Epoch:  30% 3/10 [16:48<39:10, 335.72s/it]04/09/2022 07:56:31 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 07:56:31 - INFO - __main__ -   Loss: 0.030061066150665283\n","04/09/2022 08:01:45 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 08:01:45 - INFO - __main__ -   Loss: 0.06899362847815592\n","04/09/2022 08:02:04 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:02:04 - INFO - __main__ -     eval_accuracy = 0.9185779816513762\n","04/09/2022 08:02:04 - INFO - __main__ -     eval_loss = 0.2735765549282697\n","04/09/2022 08:02:04 - INFO - __main__ -     global_step = 4208\n","04/09/2022 08:02:04 - INFO - __main__ -     loss = 0.06857049832861524\n","04/09/2022 08:02:04 - INFO - __main__ -   Total acc: 0.9185779816513762\n","04/09/2022 08:02:04 - INFO - __main__ -   Best Total acc: 0.9208715596330275\n","Epoch:  40% 4/10 [22:21<33:29, 334.89s/it]04/09/2022 08:02:04 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 08:02:04 - INFO - __main__ -   Loss: 0.15422941744327545\n","04/09/2022 08:07:18 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 08:07:18 - INFO - __main__ -   Loss: 0.05004366087335554\n","04/09/2022 08:07:37 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:07:37 - INFO - __main__ -     eval_accuracy = 0.9197247706422018\n","04/09/2022 08:07:37 - INFO - __main__ -     eval_loss = 0.28293531519358706\n","04/09/2022 08:07:37 - INFO - __main__ -     global_step = 5260\n","04/09/2022 08:07:37 - INFO - __main__ -     loss = 0.04951081448455939\n","04/09/2022 08:07:37 - INFO - __main__ -   Total acc: 0.9197247706422018\n","04/09/2022 08:07:37 - INFO - __main__ -   Best Total acc: 0.9208715596330275\n","Epoch:  50% 5/10 [27:54<27:50, 334.17s/it]04/09/2022 08:07:37 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 08:07:37 - INFO - __main__ -   Loss: 0.006834646686911583\n","04/09/2022 08:12:51 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 08:12:51 - INFO - __main__ -   Loss: 0.030046618894206827\n","04/09/2022 08:13:10 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:13:10 - INFO - __main__ -     eval_accuracy = 0.9231651376146789\n","04/09/2022 08:13:10 - INFO - __main__ -     eval_loss = 0.30221292678570616\n","04/09/2022 08:13:10 - INFO - __main__ -     global_step = 6312\n","04/09/2022 08:13:10 - INFO - __main__ -     loss = 0.029826558897585585\n","04/09/2022 08:13:10 - INFO - __main__ -   Total acc: 0.9231651376146789\n","04/09/2022 08:13:11 - INFO - __main__ -   Best Total acc: 0.9231651376146789\n","Epoch:  60% 6/10 [33:28<22:16, 334.17s/it]04/09/2022 08:13:11 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 08:13:11 - INFO - __main__ -   Loss: 0.0006540404865518212\n","04/09/2022 08:18:26 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 08:18:26 - INFO - __main__ -   Loss: 0.022627103297088946\n","04/09/2022 08:18:45 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:18:45 - INFO - __main__ -     eval_accuracy = 0.9197247706422018\n","04/09/2022 08:18:45 - INFO - __main__ -     eval_loss = 0.3289600090757055\n","04/09/2022 08:18:45 - INFO - __main__ -     global_step = 7364\n","04/09/2022 08:18:45 - INFO - __main__ -     loss = 0.022465395572628055\n","04/09/2022 08:18:45 - INFO - __main__ -   Total acc: 0.9197247706422018\n","04/09/2022 08:18:45 - INFO - __main__ -   Best Total acc: 0.9231651376146789\n","Epoch:  70% 7/10 [39:02<16:42, 334.14s/it]04/09/2022 08:18:45 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 08:18:45 - INFO - __main__ -   Loss: 0.02256951667368412\n","04/09/2022 08:23:59 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 08:23:59 - INFO - __main__ -   Loss: 0.014430470535891353\n","04/09/2022 08:24:18 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:24:18 - INFO - __main__ -     eval_accuracy = 0.9185779816513762\n","04/09/2022 08:24:18 - INFO - __main__ -     eval_loss = 0.35490943132293595\n","04/09/2022 08:24:18 - INFO - __main__ -     global_step = 8416\n","04/09/2022 08:24:18 - INFO - __main__ -     loss = 0.014309298291543128\n","04/09/2022 08:24:18 - INFO - __main__ -   Total acc: 0.9185779816513762\n","04/09/2022 08:24:18 - INFO - __main__ -   Best Total acc: 0.9231651376146789\n","Epoch:  80% 8/10 [44:35<11:07, 333.73s/it]04/09/2022 08:24:18 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 08:24:18 - INFO - __main__ -   Loss: 0.0004292655212339014\n","04/09/2022 08:29:32 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 08:29:32 - INFO - __main__ -   Loss: 0.011980482454065522\n","04/09/2022 08:29:51 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:29:51 - INFO - __main__ -     eval_accuracy = 0.926605504587156\n","04/09/2022 08:29:51 - INFO - __main__ -     eval_loss = 0.3569052426758331\n","04/09/2022 08:29:51 - INFO - __main__ -     global_step = 9468\n","04/09/2022 08:29:51 - INFO - __main__ -     loss = 0.01166820231747623\n","04/09/2022 08:29:51 - INFO - __main__ -   Total acc: 0.926605504587156\n","04/09/2022 08:29:52 - INFO - __main__ -   Best Total acc: 0.926605504587156\n","Epoch:  90% 9/10 [50:09<05:33, 333.83s/it]04/09/2022 08:29:52 - INFO - __main__ -   Task: 0, Step: 0\n","04/09/2022 08:29:52 - INFO - __main__ -   Loss: 0.11118920147418976\n","04/09/2022 08:35:07 - INFO - __main__ -   Task: 0, Step: 1000\n","04/09/2022 08:35:07 - INFO - __main__ -   Loss: 0.006918985413058117\n","04/09/2022 08:35:26 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:35:26 - INFO - __main__ -     eval_accuracy = 0.9277522935779816\n","04/09/2022 08:35:26 - INFO - __main__ -     eval_loss = 0.3825365024781813\n","04/09/2022 08:35:26 - INFO - __main__ -     global_step = 10520\n","04/09/2022 08:35:26 - INFO - __main__ -     loss = 0.006858153459384448\n","04/09/2022 08:35:26 - INFO - __main__ -   Total acc: 0.9277522935779816\n","04/09/2022 08:35:27 - INFO - __main__ -   Best Total acc: 0.9277522935779816\n","Epoch: 100% 10/10 [55:44<00:00, 334.50s/it]\n","04/09/2022 08:35:30 - INFO - __main__ -   ***** Eval results *****\n","04/09/2022 08:35:30 - INFO - __main__ -     eval_accuracy = 0.9277522935779816\n","04/09/2022 08:35:30 - INFO - __main__ -     eval_loss = 0.3825365024781813\n","04/09/2022 08:35:30 - INFO - __main__ -     global_step = 10520\n","04/09/2022 08:35:30 - INFO - __main__ -     loss = 0.006858153459384448\n","04/09/2022 08:35:30 - INFO - __main__ -   Total acc: 0.9277522935779816\n"]}],"source":["!python run_multi_task.py \\\n","  --seed 42 \\\n","  --output_dir /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/Tmp_Model/sst \\\n","  --tasks single \\\n","  --task_id 0 \\\n","  --sample 'rr'\\\n","  --multi \\\n","  --do_train \\\n","  --do_eval \\\n","  --do_lower_case \\\n","  --data_dir /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/data/ \\\n","  --vocab_file /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/uncased_L-12_H-768_A-12/vocab.txt \\\n","  --bert_config_file /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/config/pals_config.json \\\n","  --init_checkpoint /content/drive/MyDrive/769/assignment3/Bert-Multi-Task2/uncased_L-12_H-768_A-12/pytorch_model.bin \\\n","  --max_seq_length 50 \\\n","  --train_batch_size 32 \\\n","  --learning_rate 2e-5 \\\n","  --num_train_epochs 10 \\\n","  --gradient_accumulation_steps 1"]}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":[],"machine_shape":"hm","name":"CS769_MTL_Multi_class.ipynb","provenance":[],"mount_file_id":"1UNF_0yoOgaux09z5cZzkc15rAgbG8Jb-","authorship_tag":"ABX9TyOKhJ1kRayVSla1re2pSjpt"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}