benchmarking-rouge-score.ipynb

{"cells":[{"cell_type":"markdown","metadata":{},"source":["## Evaluation of dhivyeshrk/bart-large-cnn-samsum on samsum"]},{"cell_type":"code","execution_count":1,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:03:43.906641Z","iopub.status.busy":"2023-12-30T08:03:43.906207Z","iopub.status.idle":"2023-12-30T08:03:50.278383Z","shell.execute_reply":"2023-12-30T08:03:50.277456Z","shell.execute_reply.started":"2023-12-30T08:03:43.906614Z"},"trusted":true},"outputs":[],"source":["from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n","import torch\n","import transformers \n","import datasets "]},{"cell_type":"markdown","metadata":{},"source":["First load the dataset"]},{"cell_type":"code","execution_count":2,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:03:50.280776Z","iopub.status.busy":"2023-12-30T08:03:50.280132Z","iopub.status.idle":"2023-12-30T08:03:50.571186Z","shell.execute_reply":"2023-12-30T08:03:50.570328Z","shell.execute_reply.started":"2023-12-30T08:03:50.280747Z"},"trusted":true},"outputs":[],"source":["import pandas as pd\n","from datasets import Dataset, DatasetDict\n","\n","df_train = pd.read_csv('/kaggle/input/samsum-dataset-text-summarization/samsum-train.csv')\n","df_test = pd.read_csv('/kaggle/input/samsum-dataset-text-summarization/samsum-test.csv')\n","df_val = pd.read_csv('/kaggle/input/samsum-dataset-text-summarization/samsum-validation.csv')\n","\n","df_train = df_train.dropna()\n","df_test = df_test.dropna()\n","df_val = df_val.dropna()"]},{"cell_type":"code","execution_count":3,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:03:50.572547Z","iopub.status.busy":"2023-12-30T08:03:50.572252Z","iopub.status.idle":"2023-12-30T08:03:50.623585Z","shell.execute_reply":"2023-12-30T08:03:50.622702Z","shell.execute_reply.started":"2023-12-30T08:03:50.572521Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["DatasetDict({\n","    train: Dataset({\n","        features: ['id', 'dialogue', 'summary', '__index_level_0__'],\n","        num_rows: 14731\n","    })\n","    test: Dataset({\n","        features: ['id', 'dialogue', 'summary'],\n","        num_rows: 819\n","    })\n","    validation: Dataset({\n","        features: ['id', 'dialogue', 'summary'],\n","        num_rows: 818\n","    })\n","})\n"]}],"source":["# Convert DataFrames to Hugging Face Datasets\n","dataset_train = Dataset.from_pandas(df_train)\n","dataset_test = Dataset.from_pandas(df_test)\n","dataset_val = Dataset.from_pandas(df_val)\n","\n","# Create DatasetDict\n","dataset_dict = DatasetDict({\n","    'train': dataset_train,\n","    'test': dataset_test,\n","    'validation': dataset_val\n","})\n","\n","print(dataset_dict)\n","dataset_samsum = dataset_dict"]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:03:50.625768Z","iopub.status.busy":"2023-12-30T08:03:50.625457Z","iopub.status.idle":"2023-12-30T08:03:50.632733Z","shell.execute_reply":"2023-12-30T08:03:50.631754Z","shell.execute_reply.started":"2023-12-30T08:03:50.625744Z"},"trusted":true},"outputs":[{"data":{"text/plain":["DatasetDict({\n","    train: Dataset({\n","        features: ['id', 'dialogue', 'summary', '__index_level_0__'],\n","        num_rows: 14731\n","    })\n","    test: Dataset({\n","        features: ['id', 'dialogue', 'summary'],\n","        num_rows: 819\n","    })\n","    validation: Dataset({\n","        features: ['id', 'dialogue', 'summary'],\n","        num_rows: 818\n","    })\n","})"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["dataset_samsum"]},{"cell_type":"code","execution_count":5,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:03:50.634589Z","iopub.status.busy":"2023-12-30T08:03:50.633972Z","iopub.status.idle":"2023-12-30T08:03:51.628145Z","shell.execute_reply":"2023-12-30T08:03:51.626987Z","shell.execute_reply.started":"2023-12-30T08:03:50.634556Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Sat Dec 30 08:03:51 2023       \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 470.161.03   Driver Version: 470.161.03   CUDA Version: 11.4     |\n","|-------------------------------+----------------------+----------------------+\n","| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n","| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n","|                               |                      |               MIG M. |\n","|===============================+======================+======================|\n","|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n","| N/A   34C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n","|                               |                      |                  N/A |\n","+-------------------------------+----------------------+----------------------+\n","                                                                               \n","+-----------------------------------------------------------------------------+\n","| Processes:                                                                  |\n","|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n","|        ID   ID                                                   Usage      |\n","|=============================================================================|\n","|  No running processes found                                                 |\n","+-----------------------------------------------------------------------------+\n"]}],"source":["!nvidia-smi"]},{"cell_type":"code","execution_count":6,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:03:51.630065Z","iopub.status.busy":"2023-12-30T08:03:51.629734Z","iopub.status.idle":"2023-12-30T08:03:51.657142Z","shell.execute_reply":"2023-12-30T08:03:51.656171Z","shell.execute_reply.started":"2023-12-30T08:03:51.630033Z"},"trusted":true},"outputs":[],"source":["device = 'cuda' if torch.cuda.is_available() else 'cpu'"]},{"cell_type":"code","execution_count":7,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:03:51.658588Z","iopub.status.busy":"2023-12-30T08:03:51.658300Z","iopub.status.idle":"2023-12-30T08:04:04.464202Z","shell.execute_reply":"2023-12-30T08:04:04.463029Z","shell.execute_reply.started":"2023-12-30T08:03:51.658564Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"22af617cdd0e42df824a16f03ff4e5fd","version_major":2,"version_minor":0},"text/plain":["tokenizer_config.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1cd6306bebcc41f2a5db86f66d15ed7a","version_major":2,"version_minor":0},"text/plain":["vocab.json:   0%|          | 0.00/798k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a0276d0cffed4a91bd0389c25c4e86a0","version_major":2,"version_minor":0},"text/plain":["merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3746fa723c564cb4a561bc744905a945","version_major":2,"version_minor":0},"text/plain":["tokenizer.json:   0%|          | 0.00/2.11M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2a18e674980c4c9b9862bfd7db3f5a7f","version_major":2,"version_minor":0},"text/plain":["special_tokens_map.json:   0%|          | 0.00/280 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"bf13b62383f742d681a37dc75dedc47b","version_major":2,"version_minor":0},"text/plain":["config.json:   0%|          | 0.00/1.66k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ba0616b8fc884df490ca1627161e90c9","version_major":2,"version_minor":0},"text/plain":["pytorch_model.bin:   0%|          | 0.00/1.63G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"568fb5d6a2e84565bea502bb0669bede","version_major":2,"version_minor":0},"text/plain":["generation_config.json:   0%|          | 0.00/358 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"}],"source":["model_ckpt = \"dhivyeshrk/bart-large-cnn-samsum\"\n","tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n","model = AutoModelForSeq2SeqLM.from_pretrained(model_ckpt).to(device)"]},{"cell_type":"code","execution_count":8,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:04:04.465781Z","iopub.status.busy":"2023-12-30T08:04:04.465474Z","iopub.status.idle":"2023-12-30T08:04:21.622313Z","shell.execute_reply":"2023-12-30T08:04:21.621275Z","shell.execute_reply.started":"2023-12-30T08:04:04.465756Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Collecting rouge_score\n","  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n","  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hRequirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from rouge_score) (1.4.0)\n","Requirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (from rouge_score) (3.2.4)\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from rouge_score) (1.24.3)\n","Requirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.10/site-packages (from rouge_score) (1.16.0)\n","Building wheels for collected packages: rouge_score\n","  Building wheel for rouge_score (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for rouge_score: filename=rouge_score-0.1.2-py3-none-any.whl size=24932 sha256=802ec01c24dd3190d0c485d452716390263ef192c1e7501efa8948441e6a9bc2\n","  Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n","Successfully built rouge_score\n","Installing collected packages: rouge_score\n","Successfully installed rouge_score-0.1.2\n"]}],"source":["!pip install rouge_score # rouge is the benchmark score for summarization models"]},{"cell_type":"code","execution_count":9,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:04:21.624073Z","iopub.status.busy":"2023-12-30T08:04:21.623789Z","iopub.status.idle":"2023-12-30T08:04:23.069810Z","shell.execute_reply":"2023-12-30T08:04:23.068989Z","shell.execute_reply.started":"2023-12-30T08:04:21.624045Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1f8c237f49584d14ac4c2420ca19eb3b","version_major":2,"version_minor":0},"text/plain":["Downloading builder script:   0%|          | 0.00/2.16k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n","  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"]}],"source":["from datasets import load_metric\n","from tqdm import tqdm\n","\n","rouge_metric = load_metric(\"rouge\")\n","rouge_names = [\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"]"]},{"cell_type":"code","execution_count":10,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:04:23.074126Z","iopub.status.busy":"2023-12-30T08:04:23.073706Z","iopub.status.idle":"2023-12-30T08:04:23.079658Z","shell.execute_reply":"2023-12-30T08:04:23.078474Z","shell.execute_reply.started":"2023-12-30T08:04:23.074092Z"},"trusted":true},"outputs":[],"source":["def chunks(list_of_elements, batch_size):\n","    \"\"\"Yield successive batch-sized chunks from list_of_elements.\"\"\" \n","    for i in range(0, len(list_of_elements), batch_size):\n","        yield list_of_elements[i : i + batch_size]"]},{"cell_type":"code","execution_count":11,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:04:23.081024Z","iopub.status.busy":"2023-12-30T08:04:23.080756Z","iopub.status.idle":"2023-12-30T08:04:23.091676Z","shell.execute_reply":"2023-12-30T08:04:23.090730Z","shell.execute_reply.started":"2023-12-30T08:04:23.081001Z"},"trusted":true},"outputs":[],"source":["def evaluate_summaries(dataset, metric, model, tokenizer, batch_size=16, device=device,\n","                                   column_text=\"article\",\n","                                   column_summary=\"highlights\"):\n","    '''Calculate respective rouge metric for the given data'''\n","    article_batches = list(chunks(dataset[column_text], batch_size)) # dialogue batches\n","    target_batches = list(chunks(dataset[column_summary], batch_size))  # target batches\n","        \n","    for article_batch, target_batch in tqdm(zip(article_batches, target_batches), total=len(article_batches)):\n","            inputs = tokenizer(article_batch, max_length=1024,  truncation=True,\n","                            padding=\"max_length\", return_tensors=\"pt\") # encode the input\n","            print(type(article_batch))\n","            summaries = model.generate(input_ids=inputs[\"input_ids\"].to(device),  # generate summary\n","                             attention_mask=inputs[\"attention_mask\"].to(device),\n","                             length_penalty=0.8, num_beams=8, max_length=128) \n","            decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,\n","                                    clean_up_tokenization_spaces=True) for s in summaries] # decode them\n","        \n","            decoded_summaries = [d.replace(\"<n>\", \" \") for d in decoded_summaries]  # misc processing\n","            metric.add_batch(predictions=decoded_summaries, references=target_batch)  # add this batch to the metric\n","        \n","    score = metric.compute() # Calculate final metric score\n","\n","    return score"]},{"cell_type":"code","execution_count":12,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:04:23.093260Z","iopub.status.busy":"2023-12-30T08:04:23.092967Z","iopub.status.idle":"2023-12-30T08:11:06.459803Z","shell.execute_reply":"2023-12-30T08:11:06.458173Z","shell.execute_reply.started":"2023-12-30T08:04:23.093234Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["  0%|          | 0/103 [00:00<?, ?it/s]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  1%|          | 1/103 [00:06<10:41,  6.29s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  2%|▏         | 2/103 [00:10<08:06,  4.82s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  3%|▎         | 3/103 [00:13<07:16,  4.36s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  4%|▍         | 4/103 [00:17<06:50,  4.14s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  5%|▍         | 5/103 [00:21<06:27,  3.96s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  6%|▌         | 6/103 [00:25<06:28,  4.01s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  7%|▋         | 7/103 [00:29<06:19,  3.96s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  8%|▊         | 8/103 [00:32<06:04,  3.83s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  9%|▊         | 9/103 [00:36<06:03,  3.87s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 10%|▉         | 10/103 [00:40<05:55,  3.82s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 11%|█         | 11/103 [00:44<05:53,  3.84s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 12%|█▏        | 12/103 [00:48<05:49,  3.84s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 13%|█▎        | 13/103 [00:51<05:41,  3.79s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 14%|█▎        | 14/103 [00:55<05:34,  3.76s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 15%|█▍        | 15/103 [00:59<05:41,  3.88s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 16%|█▌        | 16/103 [01:03<05:33,  3.83s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 17%|█▋        | 17/103 [01:07<05:30,  3.84s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 17%|█▋        | 18/103 [01:11<05:22,  3.80s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 18%|█▊        | 19/103 [01:14<05:14,  3.75s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 19%|█▉        | 20/103 [01:18<05:11,  3.75s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 20%|██        | 21/103 [01:22<05:09,  3.78s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 21%|██▏       | 22/103 [01:26<05:05,  3.78s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 22%|██▏       | 23/103 [01:29<05:02,  3.78s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 23%|██▎       | 24/103 [01:33<05:01,  3.81s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 24%|██▍       | 25/103 [01:37<05:03,  3.89s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 25%|██▌       | 26/103 [01:41<04:58,  3.87s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 26%|██▌       | 27/103 [01:45<04:50,  3.83s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 27%|██▋       | 28/103 [01:49<04:54,  3.92s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 28%|██▊       | 29/103 [01:53<04:50,  3.92s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 29%|██▉       | 30/103 [01:57<04:41,  3.86s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 30%|███       | 31/103 [02:00<04:36,  3.85s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 31%|███       | 32/103 [02:04<04:36,  3.90s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 32%|███▏      | 33/103 [02:09<04:38,  3.98s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 33%|███▎      | 34/103 [02:12<04:30,  3.91s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 34%|███▍      | 35/103 [02:16<04:23,  3.88s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 35%|███▍      | 36/103 [02:20<04:24,  3.95s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 36%|███▌      | 37/103 [02:24<04:20,  3.95s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 37%|███▋      | 38/103 [02:28<04:18,  3.98s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 38%|███▊      | 39/103 [02:32<04:09,  3.89s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 39%|███▉      | 40/103 [02:36<04:00,  3.81s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 40%|███▉      | 41/103 [02:40<03:59,  3.87s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 41%|████      | 42/103 [02:43<03:53,  3.82s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 42%|████▏     | 43/103 [02:47<03:48,  3.82s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 43%|████▎     | 44/103 [02:53<04:17,  4.37s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 44%|████▎     | 45/103 [02:57<04:02,  4.18s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 45%|████▍     | 46/103 [03:00<03:50,  4.04s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 46%|████▌     | 47/103 [03:04<03:43,  4.00s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 47%|████▋     | 48/103 [03:08<03:36,  3.93s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 48%|████▊     | 49/103 [03:12<03:29,  3.89s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 49%|████▊     | 50/103 [03:16<03:27,  3.92s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 50%|████▉     | 51/103 [03:20<03:21,  3.88s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 50%|█████     | 52/103 [03:24<03:20,  3.94s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 51%|█████▏    | 53/103 [03:28<03:23,  4.06s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 52%|█████▏    | 54/103 [03:32<03:22,  4.13s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 53%|█████▎    | 55/103 [03:36<03:15,  4.07s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 54%|█████▍    | 56/103 [03:40<03:07,  3.98s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 55%|█████▌    | 57/103 [03:44<02:59,  3.91s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 56%|█████▋    | 58/103 [03:47<02:54,  3.88s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 57%|█████▋    | 59/103 [03:51<02:52,  3.91s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 58%|█████▊    | 60/103 [03:55<02:49,  3.94s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 59%|█████▉    | 61/103 [03:59<02:42,  3.86s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 60%|██████    | 62/103 [04:03<02:36,  3.83s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 61%|██████    | 63/103 [04:07<02:35,  3.89s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 62%|██████▏   | 64/103 [04:11<02:32,  3.91s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 63%|██████▎   | 65/103 [04:15<02:26,  3.86s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 64%|██████▍   | 66/103 [04:18<02:20,  3.81s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 65%|██████▌   | 67/103 [04:22<02:16,  3.80s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 66%|██████▌   | 68/103 [04:26<02:14,  3.86s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 67%|██████▋   | 69/103 [04:30<02:08,  3.79s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 68%|██████▊   | 70/103 [04:33<02:04,  3.77s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 69%|██████▉   | 71/103 [04:37<01:59,  3.74s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 70%|██████▉   | 72/103 [04:41<01:55,  3.71s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 71%|███████   | 73/103 [04:45<01:52,  3.74s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 72%|███████▏  | 74/103 [04:48<01:50,  3.80s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 73%|███████▎  | 75/103 [04:52<01:45,  3.78s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 74%|███████▍  | 76/103 [04:56<01:43,  3.83s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 75%|███████▍  | 77/103 [05:00<01:40,  3.86s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 76%|███████▌  | 78/103 [05:04<01:35,  3.81s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 77%|███████▋  | 79/103 [05:08<01:31,  3.80s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 78%|███████▊  | 80/103 [05:12<01:28,  3.85s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 79%|███████▊  | 81/103 [05:16<01:25,  3.91s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 80%|███████▉  | 82/103 [05:21<01:30,  4.31s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 81%|████████  | 83/103 [05:25<01:22,  4.13s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 82%|████████▏ | 84/103 [05:29<01:17,  4.08s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 83%|████████▎ | 85/103 [05:32<01:12,  4.02s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 83%|████████▎ | 86/103 [05:36<01:07,  3.99s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 84%|████████▍ | 87/103 [05:40<01:03,  3.95s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 85%|████████▌ | 88/103 [05:44<00:59,  3.94s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 86%|████████▋ | 89/103 [05:49<00:58,  4.15s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 87%|████████▋ | 90/103 [05:52<00:51,  4.00s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 88%|████████▊ | 91/103 [05:56<00:46,  3.90s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 89%|████████▉ | 92/103 [06:00<00:43,  3.95s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 90%|█████████ | 93/103 [06:04<00:38,  3.87s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 91%|█████████▏| 94/103 [06:08<00:35,  3.93s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 92%|█████████▏| 95/103 [06:12<00:30,  3.87s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 93%|█████████▎| 96/103 [06:16<00:28,  4.11s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 94%|█████████▍| 97/103 [06:20<00:24,  4.08s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 95%|█████████▌| 98/103 [06:24<00:19,  3.99s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 96%|█████████▌| 99/103 [06:28<00:15,  3.93s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 97%|█████████▋| 100/103 [06:32<00:11,  3.95s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 98%|█████████▊| 101/103 [06:36<00:07,  3.90s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 99%|█████████▉| 102/103 [06:40<00:03,  3.90s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["100%|██████████| 103/103 [06:41<00:00,  3.90s/it]\n"]}],"source":["score_fineTuned = evaluate_summaries(dataset_samsum[\"test\"], rouge_metric, model, tokenizer, column_text=\"dialogue\", column_summary=\"summary\", batch_size=8)\n","rouge_dict_fineTuned = dict((rn, score_fineTuned[rn].mid.fmeasure) for rn in rouge_names)"]},{"cell_type":"code","execution_count":14,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:15:57.013456Z","iopub.status.busy":"2023-12-30T08:15:57.013022Z","iopub.status.idle":"2023-12-30T08:15:57.029100Z","shell.execute_reply":"2023-12-30T08:15:57.028182Z","shell.execute_reply.started":"2023-12-30T08:15:57.013423Z"},"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>rouge1</th>\n","      <th>rouge2</th>\n","      <th>rougeL</th>\n","      <th>rougeLsum</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>bart-cnn-FineTuned</th>\n","      <td>0.392008</td>\n","      <td>0.197664</td>\n","      <td>0.30173</td>\n","      <td>0.30199</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                      rouge1    rouge2   rougeL  rougeLsum\n","bart-cnn-FineTuned  0.392008  0.197664  0.30173    0.30199"]},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":["pd.DataFrame.from_records(rouge_dict_fineTuned, index=[f\"bart-cnn-FineTuned\"])"]},{"cell_type":"code","execution_count":21,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:26:19.812888Z","iopub.status.busy":"2023-12-30T08:26:19.812480Z","iopub.status.idle":"2023-12-30T08:26:20.346387Z","shell.execute_reply":"2023-12-30T08:26:20.345262Z","shell.execute_reply.started":"2023-12-30T08:26:19.812858Z"},"trusted":true},"outputs":[],"source":["torch.cuda.empty_cache()  # Free GPU Memory to test on original model. "]},{"cell_type":"code","execution_count":22,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:27:13.904757Z","iopub.status.busy":"2023-12-30T08:27:13.903936Z","iopub.status.idle":"2023-12-30T08:27:23.097339Z","shell.execute_reply":"2023-12-30T08:27:23.096486Z","shell.execute_reply.started":"2023-12-30T08:27:13.904719Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"609b78eb5e2647b38f409767d4f50057","version_major":2,"version_minor":0},"text/plain":["config.json:   0%|          | 0.00/1.58k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"529547cb341b49089098145e3c02752e","version_major":2,"version_minor":0},"text/plain":["vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"cb2e3ad00839418ab6fbbf8180b0de6c","version_major":2,"version_minor":0},"text/plain":["merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b1c90cdec5d94d0ca221157e3d7dfccb","version_major":2,"version_minor":0},"text/plain":["tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"55d556b91fe54d0c841e22346e9a9281","version_major":2,"version_minor":0},"text/plain":["model.safetensors:   0%|          | 0.00/1.63G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0ff4d67136024063b309201fb8be2801","version_major":2,"version_minor":0},"text/plain":["generation_config.json:   0%|          | 0.00/363 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"}],"source":["model_ckpt = \"facebook/bart-large-cnn\"\n","tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n","model = AutoModelForSeq2SeqLM.from_pretrained(model_ckpt).to(device)"]},{"cell_type":"code","execution_count":23,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:27:23.099804Z","iopub.status.busy":"2023-12-30T08:27:23.099191Z","iopub.status.idle":"2023-12-30T08:35:21.053410Z","shell.execute_reply":"2023-12-30T08:35:21.052445Z","shell.execute_reply.started":"2023-12-30T08:27:23.099766Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["  0%|          | 0/103 [00:00<?, ?it/s]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  1%|          | 1/103 [00:04<08:03,  4.74s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  2%|▏         | 2/103 [00:08<07:28,  4.44s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  3%|▎         | 3/103 [00:13<07:10,  4.31s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  4%|▍         | 4/103 [00:18<08:02,  4.87s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  5%|▍         | 5/103 [00:23<07:52,  4.82s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  6%|▌         | 6/103 [00:28<07:44,  4.79s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  7%|▋         | 7/103 [00:32<07:35,  4.74s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  8%|▊         | 8/103 [00:37<07:12,  4.55s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["  9%|▊         | 9/103 [00:41<07:06,  4.53s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 10%|▉         | 10/103 [00:46<07:14,  4.67s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 11%|█         | 11/103 [00:51<07:11,  4.69s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 12%|█▏        | 12/103 [00:59<08:32,  5.63s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 13%|█▎        | 13/103 [01:03<07:47,  5.20s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 14%|█▎        | 14/103 [01:07<07:16,  4.90s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 15%|█▍        | 15/103 [01:12<07:08,  4.87s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 16%|█▌        | 16/103 [01:16<06:43,  4.64s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 17%|█▋        | 17/103 [01:20<06:25,  4.48s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 17%|█▋        | 18/103 [01:25<06:24,  4.52s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 18%|█▊        | 19/103 [01:32<07:24,  5.30s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 19%|█▉        | 20/103 [01:37<07:25,  5.37s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 20%|██        | 21/103 [01:41<06:51,  5.02s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 21%|██▏       | 22/103 [01:46<06:42,  4.97s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 22%|██▏       | 23/103 [01:51<06:27,  4.85s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 23%|██▎       | 24/103 [01:56<06:17,  4.78s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 24%|██▍       | 25/103 [02:00<06:02,  4.64s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 25%|██▌       | 26/103 [02:05<06:01,  4.69s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 26%|██▌       | 27/103 [02:09<05:56,  4.69s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 27%|██▋       | 28/103 [02:14<05:43,  4.58s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 28%|██▊       | 29/103 [02:18<05:36,  4.55s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 29%|██▉       | 30/103 [02:22<05:23,  4.43s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 30%|███       | 31/103 [02:26<05:11,  4.32s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 31%|███       | 32/103 [02:32<05:26,  4.60s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 32%|███▏      | 33/103 [02:36<05:24,  4.63s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 33%|███▎      | 34/103 [02:41<05:15,  4.57s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 34%|███▍      | 35/103 [02:45<04:57,  4.38s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 35%|███▍      | 36/103 [02:49<04:53,  4.39s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 36%|███▌      | 37/103 [02:54<05:01,  4.57s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 37%|███▋      | 38/103 [02:59<04:56,  4.55s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 38%|███▊      | 39/103 [03:03<04:52,  4.56s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 39%|███▉      | 40/103 [03:08<04:46,  4.55s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 40%|███▉      | 41/103 [03:12<04:34,  4.43s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 41%|████      | 42/103 [03:16<04:30,  4.43s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 42%|████▏     | 43/103 [03:21<04:26,  4.45s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 43%|████▎     | 44/103 [03:27<04:49,  4.90s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 44%|████▎     | 45/103 [03:32<04:44,  4.90s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 45%|████▍     | 46/103 [03:36<04:28,  4.71s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 46%|████▌     | 47/103 [03:40<04:13,  4.53s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 47%|████▋     | 48/103 [03:44<04:05,  4.46s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 48%|████▊     | 49/103 [03:49<04:12,  4.68s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 49%|████▊     | 50/103 [03:54<04:03,  4.60s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 50%|████▉     | 51/103 [03:58<03:55,  4.53s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 50%|█████     | 52/103 [04:03<03:46,  4.45s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 51%|█████▏    | 53/103 [04:07<03:45,  4.51s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 52%|█████▏    | 54/103 [04:12<03:41,  4.53s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 53%|█████▎    | 55/103 [04:16<03:35,  4.50s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 54%|█████▍    | 56/103 [04:20<03:28,  4.44s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 55%|█████▌    | 57/103 [04:25<03:22,  4.39s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 56%|█████▋    | 58/103 [04:29<03:15,  4.34s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 57%|█████▋    | 59/103 [04:33<03:13,  4.39s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 58%|█████▊    | 60/103 [04:38<03:07,  4.36s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 59%|█████▉    | 61/103 [04:42<03:01,  4.33s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 60%|██████    | 62/103 [04:46<02:55,  4.28s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 61%|██████    | 63/103 [04:51<02:58,  4.47s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 62%|██████▏   | 64/103 [04:55<02:51,  4.41s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 63%|██████▎   | 65/103 [05:00<02:46,  4.39s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 64%|██████▍   | 66/103 [05:04<02:42,  4.39s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 65%|██████▌   | 67/103 [05:08<02:36,  4.36s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 66%|██████▌   | 68/103 [05:13<02:35,  4.45s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 67%|██████▋   | 69/103 [05:20<03:01,  5.33s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 68%|██████▊   | 70/103 [05:25<02:48,  5.10s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 69%|██████▉   | 71/103 [05:30<02:40,  5.00s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 70%|██████▉   | 72/103 [05:34<02:28,  4.80s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 71%|███████   | 73/103 [05:42<02:48,  5.62s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 72%|███████▏  | 74/103 [05:46<02:33,  5.29s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 73%|███████▎  | 75/103 [05:51<02:26,  5.24s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 74%|███████▍  | 76/103 [05:56<02:13,  4.94s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 75%|███████▍  | 77/103 [06:00<02:04,  4.78s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 76%|███████▌  | 78/103 [06:04<01:56,  4.67s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 77%|███████▋  | 79/103 [06:08<01:47,  4.47s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 78%|███████▊  | 80/103 [06:13<01:43,  4.51s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 79%|███████▊  | 81/103 [06:18<01:41,  4.60s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 80%|███████▉  | 82/103 [06:23<01:40,  4.80s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 81%|████████  | 83/103 [06:27<01:31,  4.58s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 82%|████████▏ | 84/103 [06:32<01:26,  4.57s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 83%|████████▎ | 85/103 [06:36<01:21,  4.54s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 83%|████████▎ | 86/103 [06:41<01:18,  4.63s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 84%|████████▍ | 87/103 [06:46<01:13,  4.61s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 85%|████████▌ | 88/103 [06:50<01:09,  4.64s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 86%|████████▋ | 89/103 [06:55<01:04,  4.63s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 87%|████████▋ | 90/103 [06:59<01:00,  4.63s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 88%|████████▊ | 91/103 [07:04<00:54,  4.57s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 89%|████████▉ | 92/103 [07:08<00:49,  4.50s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 90%|█████████ | 93/103 [07:12<00:44,  4.42s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 91%|█████████▏| 94/103 [07:18<00:41,  4.64s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 92%|█████████▏| 95/103 [07:22<00:36,  4.50s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 93%|█████████▎| 96/103 [07:26<00:31,  4.43s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 94%|█████████▍| 97/103 [07:31<00:28,  4.70s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 95%|█████████▌| 98/103 [07:36<00:23,  4.63s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 96%|█████████▌| 99/103 [07:40<00:18,  4.59s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 97%|█████████▋| 100/103 [07:45<00:13,  4.51s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 98%|█████████▊| 101/103 [07:49<00:08,  4.50s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":[" 99%|█████████▉| 102/103 [07:54<00:04,  4.64s/it]"]},{"name":"stdout","output_type":"stream","text":["<class 'list'>\n"]},{"name":"stderr","output_type":"stream","text":["100%|██████████| 103/103 [07:56<00:00,  4.62s/it]\n"]}],"source":["score_original = evaluate_summaries(dataset_samsum[\"test\"], rouge_metric, model, tokenizer, column_text=\"dialogue\", column_summary=\"summary\", batch_size=8)\n","rouge_dict_original = dict((rn, score_original[rn].mid.fmeasure) for rn in rouge_names)"]},{"cell_type":"code","execution_count":24,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:35:21.055041Z","iopub.status.busy":"2023-12-30T08:35:21.054690Z","iopub.status.idle":"2023-12-30T08:35:21.066226Z","shell.execute_reply":"2023-12-30T08:35:21.065354Z","shell.execute_reply.started":"2023-12-30T08:35:21.055008Z"},"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>rouge1</th>\n","      <th>rouge2</th>\n","      <th>rougeL</th>\n","      <th>rougeLsum</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>bart-cnn-Facebook</th>\n","      <td>0.300117</td>\n","      <td>0.097893</td>\n","      <td>0.226542</td>\n","      <td>0.226479</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                     rouge1    rouge2    rougeL  rougeLsum\n","bart-cnn-Facebook  0.300117  0.097893  0.226542   0.226479"]},"execution_count":24,"metadata":{},"output_type":"execute_result"}],"source":["pd.DataFrame.from_records(rouge_dict_original, index=[f\"bart-cnn-Facebook\"])"]},{"cell_type":"code","execution_count":25,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T08:46:12.923396Z","iopub.status.busy":"2023-12-30T08:46:12.922415Z","iopub.status.idle":"2023-12-30T08:46:13.972277Z","shell.execute_reply":"2023-12-30T08:46:13.971100Z","shell.execute_reply.started":"2023-12-30T08:46:12.923358Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n","To disable this warning, you can either:\n","\t- Avoid using `tokenizers` before the fork if possible\n","\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"]},{"name":"stdout","output_type":"stream","text":["Sat Dec 30 08:46:13 2023       \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 470.161.03   Driver Version: 470.161.03   CUDA Version: 11.4     |\n","|-------------------------------+----------------------+----------------------+\n","| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n","| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n","|                               |                      |               MIG M. |\n","|===============================+======================+======================|\n","|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n","| N/A   36C    P0    33W / 250W |  14445MiB / 16280MiB |      0%      Default |\n","|                               |                      |                  N/A |\n","+-------------------------------+----------------------+----------------------+\n","                                                                               \n","+-----------------------------------------------------------------------------+\n","| Processes:                                                                  |\n","|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n","|        ID   ID                                                   Usage      |\n","|=============================================================================|\n","+-----------------------------------------------------------------------------+\n"]}],"source":["!nvidia-smi"]},{"cell_type":"markdown","metadata":{},"source":["Now we can test the fine-tuned model"]},{"cell_type":"code","execution_count":43,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T09:12:48.844779Z","iopub.status.busy":"2023-12-30T09:12:48.844015Z","iopub.status.idle":"2023-12-30T09:12:50.575484Z","shell.execute_reply":"2023-12-30T09:12:50.574678Z","shell.execute_reply.started":"2023-12-30T09:12:48.844743Z"},"trusted":true},"outputs":[],"source":["from transformers import pipeline\n","model_cktpt = 'dhivyeshrk/bart-large-cnn-samsum'\n","gen_kwargs = {\"length_penalty\": 0.9, \n","             \"num_beams\": 8,\n","             \"max_length\": 64}\n","# sample_text = dataset_samsum[\"test\"][0][\"dialogue\"]\n","# reference = dataset_samsum[\"test\"][0][\"summary\"]\n","pipe = pipeline(\"summarization\", model=model_ckpt)"]},{"cell_type":"code","execution_count":46,"metadata":{"execution":{"iopub.execute_input":"2023-12-30T09:13:49.941895Z","iopub.status.busy":"2023-12-30T09:13:49.940942Z","iopub.status.idle":"2023-12-30T09:13:58.464514Z","shell.execute_reply":"2023-12-30T09:13:58.463447Z","shell.execute_reply.started":"2023-12-30T09:13:49.941861Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["The telecom industry is going through a rough patch because of falling prices and shrinking margins. Even if the job remains, career growth isn’t exciting. Mahesh is leaning toward digital marketing because in that industry he can carry over some of his skills from the current job.\n"]}],"source":["custom_dialogue = \"\"\"\\\n","Rohit: You look bit down. What’s the matter?\n","Mahesh: (Sighs) Nothing much.\n","Rohit: Looks like something isn’t right.\n","Mahesh: Ya. It’s at the job front. You know that the telecom industry is going through a rough patch because of\n","        falling prices and shrinking margins. These factors along with consolidation in the industry is\n","        threatening the stability of our jobs. And even if the job remains, career growth isn’t exciting.\n","Rohit: I know. I’ve been reading about some of these issues about your industry in the newspapers. \n","        So have you thought of any plan?\n","Mahesh: I’ve been thinking about it for a while, but haven’t concretized anything so far.\n","Rohit: What have you been thinking, if you can share?\n","Mahesh: Well, I’ve been thinking of switching to an industry that has at least few decades of growth left.\n","Rohit: That’s the right approach, but you need to reskill yourself for the industry you’re targeting.\n","Mahesh: I realize that, and I’ve been leaning toward digital marketing because in that industry I can \n","        carry over some of my skills from the current job. Another reason for this inclination \n","        is that digital marketing requires far less hardcore technical skills, which will make it relatively \n","        easier for me to acquire new skills.\n","Rohit: Your choice makes sense. So are you thinking of making the transition in near future?\n","Mahesh: Not immediately. I need to keep the job, as I’ve EMIs to pay. I’m 80-90 percent sure I’ll go with \n","        digital marketing as the industry to reskill in, but in the next 2-3 weeks I’ll take more\n","        opinions on other options, after all I wouldn’t want to change the industry again. \n","        And once I finalize the industry, I’ll explore different options to reskill while keeping my current job.\n","Rohit: Sounds like a plan. If you need I can put you in touch with few friends who can help you finalize \n","        your future industry.\n","\"\"\"\n","\n","# print(pipe(custom_dialogue, **gen_kwargs)[0][\"summary_text\"])\n","print(pipe(custom_dialogue)[0][\"summary_text\"])"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"datasetId":3438844,"sourceId":6004344,"sourceType":"datasetVersion"}],"dockerImageVersionId":30627,"isGpuEnabled":true,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat":4,"nbformat_minor":4}