Sarcasm-Detection-LLaMA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA SETUP: CUDA runtime path found: /home/user/ishfaqm0/miniconda3/envs/nlp2/lib/libcudart.so.11.0\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n",
      "[2023-07-04 20:09:04,541] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import torch\n",
    "\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,2,3,4,5\"\n",
    "\n",
    "# Check if GPU is available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Using device:\", device)\n",
    "\n",
    "import transformers\n",
    "import textwrap\n",
    "from transformers import LlamaTokenizer, LlamaForCausalLM\n",
    "\n",
    "import sys\n",
    "from typing import List\n",
    "\n",
    "from peft import (\n",
    "    LoraConfig,\n",
    "    get_peft_model,\n",
    "    get_peft_model_state_dict,\n",
    "    prepare_model_for_int8_training,\n",
    ")\n",
    "\n",
    "import fire\n",
    "\n",
    "from datasets import load_dataset\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "import json\n",
    "import numpy as np\n",
    "import wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    }
   ],
   "source": [
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,2,3,4,5\"\n",
    "\n",
    "# Check if GPU is available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Using device:\", device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_random_seed(random_seed=None):\n",
    "    \"\"\"\n",
    "    Using random seed for numpy and torch\n",
    "    \"\"\"\n",
    "    if random_seed is None:\n",
    "        random_seed = 13\n",
    "    os.environ[\"PY/THONHASHSEED\"] = str(random_seed)\n",
    "    np.random.seed(random_seed)\n",
    "    torch.manual_seed(random_seed)\n",
    "    torch.cuda.manual_seed_all(random_seed)\n",
    "\n",
    "\n",
    "set_random_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sarcasm_df = pd.read_csv(\"data/train-balanced-sarcasm.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>comment</th>\n",
       "      <th>author</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>score</th>\n",
       "      <th>ups</th>\n",
       "      <th>downs</th>\n",
       "      <th>date</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>parent_comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NC and NH.</td>\n",
       "      <td>Trumpbart</td>\n",
       "      <td>politics</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-10</td>\n",
       "      <td>2016-10-16 23:55:23</td>\n",
       "      <td>Yeah, I get that argument. At this point, I'd ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>You do know west teams play against west teams...</td>\n",
       "      <td>Shbshb906</td>\n",
       "      <td>nba</td>\n",
       "      <td>-4</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-11</td>\n",
       "      <td>2016-11-01 00:24:10</td>\n",
       "      <td>The blazers and Mavericks (The wests 5 and 6 s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>They were underdogs earlier today, but since G...</td>\n",
       "      <td>Creepeth</td>\n",
       "      <td>nfl</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-09</td>\n",
       "      <td>2016-09-22 21:45:37</td>\n",
       "      <td>They're favored to win.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>This meme isn't funny none of the \"new york ni...</td>\n",
       "      <td>icebrotha</td>\n",
       "      <td>BlackPeopleTwitter</td>\n",
       "      <td>-8</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-10</td>\n",
       "      <td>2016-10-18 21:03:47</td>\n",
       "      <td>deadass don't kill my buzz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>I could use one of those tools.</td>\n",
       "      <td>cush2push</td>\n",
       "      <td>MaddenUltimateTeam</td>\n",
       "      <td>6</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-12</td>\n",
       "      <td>2016-12-30 17:00:13</td>\n",
       "      <td>Yep can confirm I saw the tool they use for th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                            comment     author  \\\n",
       "0      0                                         NC and NH.  Trumpbart   \n",
       "1      0  You do know west teams play against west teams...  Shbshb906   \n",
       "2      0  They were underdogs earlier today, but since G...   Creepeth   \n",
       "3      0  This meme isn't funny none of the \"new york ni...  icebrotha   \n",
       "4      0                    I could use one of those tools.  cush2push   \n",
       "\n",
       "            subreddit  score  ups  downs     date          created_utc  \\\n",
       "0            politics      2   -1     -1  2016-10  2016-10-16 23:55:23   \n",
       "1                 nba     -4   -1     -1  2016-11  2016-11-01 00:24:10   \n",
       "2                 nfl      3    3      0  2016-09  2016-09-22 21:45:37   \n",
       "3  BlackPeopleTwitter     -8   -1     -1  2016-10  2016-10-18 21:03:47   \n",
       "4  MaddenUltimateTeam      6   -1     -1  2016-12  2016-12-30 17:00:13   \n",
       "\n",
       "                                      parent_comment  \n",
       "0  Yeah, I get that argument. At this point, I'd ...  \n",
       "1  The blazers and Mavericks (The wests 5 and 6 s...  \n",
       "2                            They're favored to win.  \n",
       "3                         deadass don't kill my buzz  \n",
       "4  Yep can confirm I saw the tool they use for th...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sarcasm_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We just need comment & label columns\n",
    "# So, let's remove others.\n",
    "sarcasm_df.drop(\n",
    "    [\n",
    "        \"author\",\n",
    "        \"subreddit\",\n",
    "        \"score\",\n",
    "        \"ups\",\n",
    "        \"downs\",\n",
    "        \"date\",\n",
    "        \"created_utc\",\n",
    "        \"parent_comment\",\n",
    "    ],\n",
    "    axis=1,\n",
    "    inplace=True,\n",
    ")\n",
    "# remove empty rows\n",
    "sarcasm_df.dropna(inplace=True)\n",
    "\n",
    "# Some comments are missing, so we drop the corresponding rows.\n",
    "sarcasm_df.dropna(subset=[\"comment\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label\n",
       "0    505403\n",
       "1    505368\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sarcasm_df[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean length: 10.461467533199905\n",
      "Maximum length: 2222\n",
      "Minimum length: 1\n"
     ]
    }
   ],
   "source": [
    "# Calculate the lengths of comments\n",
    "comment_lengths = [len(comment.split()) for comment in sarcasm_df[\"comment\"]]\n",
    "\n",
    "# Calculate the mean, maximum, and minimum lengths\n",
    "mean_length = sum(comment_lengths) / len(comment_lengths)\n",
    "max_length = max(comment_lengths)\n",
    "min_length = min(comment_lengths)\n",
    "\n",
    "# Print the results\n",
    "print(\"Mean length:\", mean_length)\n",
    "print(\"Maximum length:\", max_length)\n",
    "print(\"Minimum length:\", min_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter the dataframe to keep only comments with length <= 50\n",
    "mask = [length <= 50 for length in comment_lengths]\n",
    "sarcasm_df = sarcasm_df[mask]\n",
    "\n",
    "# Reset the index of the dataframe\n",
    "sarcasm_df.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean length: 10.265359705412772\n",
      "Maximum length: 50\n",
      "Minimum length: 1\n"
     ]
    }
   ],
   "source": [
    "# Calculate the lengths of comments\n",
    "comment_lengths = [len(comment.split()) for comment in sarcasm_df[\"comment\"]]\n",
    "\n",
    "# Calculate the mean, maximum, and minimum lengths\n",
    "mean_length = sum(comment_lengths) / len(comment_lengths)\n",
    "max_length = max(comment_lengths)\n",
    "min_length = min(comment_lengths)\n",
    "\n",
    "# Print the results\n",
    "print(\"Mean length:\", mean_length)\n",
    "print(\"Maximum length:\", max_length)\n",
    "print(\"Minimum length:\", min_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NC and NH.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>You do know west teams play against west teams...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>They were underdogs earlier today, but since G...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>This meme isn't funny none of the \"new york ni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>I could use one of those tools.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                            comment\n",
       "0      0                                         NC and NH.\n",
       "1      0  You do know west teams play against west teams...\n",
       "2      0  They were underdogs earlier today, but since G...\n",
       "3      0  This meme isn't funny none of the \"new york ni...\n",
       "4      0                    I could use one of those tools."
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sarcasm_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label\n",
       "1    504617\n",
       "0    503166\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sarcasm_df[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='label'>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGrCAYAAAAsBPjXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAk1ElEQVR4nO3df2xVB/3/8VdbuLf8uheB0ruGMjDMQR0/pEC56qZzlbvZLZKVCJNsHTAnpODodVBQUiaaD4SpFORH1UW7mBGBP4YblSIWYW7cDSh2A7Q4laWYetvi1l7od7Sl7fcP0yN3IG1h9ELfz0dyk/We9z19t/HKM5d7D3Ht7e3tAgAAMCg+1gsAAADECiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmNUn1gvcytra2lRdXa1BgwYpLi4u1usAAIAuaG9v1/nz55WSkqL4+Gu/5kMIXUN1dbVSU1NjvQYAALgOZ8+e1YgRI645Qwhdw6BBgyT95xfp8XhivA0AAOiKSCSi1NRU58/xayGErqHjr8M8Hg8hBADAbaYrb2vhzdIAAMAsQggAAJhFCAEAALMIIQAAYBYhBAAAzCKEAACAWYQQAAAwixACAABmEUIAAMAsQggAAJhFCAEAALMIIQAAYBYhBAAAzCKEAACAWd0Koeeee05xcXFRt7FjxzrHL168qNzcXA0dOlQDBw5Udna2ampqos5RVVWlrKws9e/fX8OHD9eyZct06dKlqJmDBw9q8uTJcrvdGjNmjIqLi6/YZcuWLRo1apQSExOVkZGhI0eORB3vyi4AAMC2Pt19wKc//Wn9/ve//+8J+vz3FHl5eSopKdGuXbvk9Xq1ePFiPfroo3rjjTckSa2trcrKypLP59Phw4f1r3/9S0888YT69u2r//u//5MknTlzRllZWVq4cKFeeukllZWV6amnntIdd9yhQCAgSdqxY4eCwaCKioqUkZGhwsJCBQIBnT59WsOHD+/SLri2UStKYr0CetB767JivQIAxERce3t7e1eHn3vuOe3evVsVFRVXHGtoaFBSUpK2b9+uWbNmSZIqKys1btw4hUIhTZ8+XXv37tXDDz+s6upqJScnS5KKioqUn5+vuro6uVwu5efnq6SkRCdPnnTOPWfOHNXX16u0tFSSlJGRoalTp2rz5s2SpLa2NqWmpmrJkiVasWJFl3bpikgkIq/Xq4aGBnk8nq7+mnoFQsgWQsgWnt+2WHx+d+fP726/R+jdd99VSkqKPvnJT2ru3LmqqqqSJJWXl6ulpUWZmZnO7NixYzVy5EiFQiFJUigU0vjx450IkqRAIKBIJKJTp045M5efo2Om4xzNzc0qLy+PmomPj1dmZqYz05VdrqapqUmRSCTqBgAAeq9uhVBGRoaKi4tVWlqqbdu26cyZM7r33nt1/vx5hcNhuVwuDR48OOoxycnJCofDkqRwOBwVQR3HO45dayYSiejDDz/UuXPn1NraetWZy8/R2S5Xs3btWnm9XueWmpratV8MAAC4LXXrPUIPPfSQ898TJkxQRkaG7rzzTu3cuVP9+vX72JfraStXrlQwGHS+jkQixBAAAL3YDX18fvDgwfrUpz6lv/3tb/L5fGpublZ9fX3UTE1NjXw+nyTJ5/Nd8cmtjq87m/F4POrXr5+GDRumhISEq85cfo7Odrkat9stj8cTdQMAAL3XDYXQhQsX9Pe//1133HGH0tPT1bdvX5WVlTnHT58+raqqKvn9fkmS3+/XiRMnVFtb68zs379fHo9HaWlpzszl5+iY6TiHy+VSenp61ExbW5vKysqcma7sAgAA0K2/Gnv22Wf1yCOP6M4771R1dbVWr16thIQEPfbYY/J6vVqwYIGCwaCGDBkij8ejJUuWyO/3O5/SmjFjhtLS0vT4449r/fr1CofDWrVqlXJzc+V2uyVJCxcu1ObNm7V8+XLNnz9fBw4c0M6dO1VS8t9POQSDQeXk5GjKlCmaNm2aCgsL1djYqHnz5klSl3YBAADoVgj985//1GOPPaZ///vfSkpK0uc//3m9+eabSkpKkiRt2LBB8fHxys7OVlNTkwKBgLZu3eo8PiEhQXv27NGiRYvk9/s1YMAA5eTkaM2aNc7M6NGjVVJSory8PG3cuFEjRozQCy+84FxDSJJmz56turo6FRQUKBwOa9KkSSotLY16A3VnuwAAAHTrOkLWcB0hWGHxOiOW8fy2xeLz+6ZeRwgAAKC3IIQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADDrhkJo3bp1iouL09KlS537Ll68qNzcXA0dOlQDBw5Udna2ampqoh5XVVWlrKws9e/fX8OHD9eyZct06dKlqJmDBw9q8uTJcrvdGjNmjIqLi6/4/lu2bNGoUaOUmJiojIwMHTlyJOp4V3YBAAB2XXcIHT16VD/96U81YcKEqPvz8vL06quvateuXTp06JCqq6v16KOPOsdbW1uVlZWl5uZmHT58WC+++KKKi4tVUFDgzJw5c0ZZWVm6//77VVFRoaVLl+qpp57Svn37nJkdO3YoGAxq9erVOn78uCZOnKhAIKDa2tou7wIAAGyLa29vb+/ugy5cuKDJkydr69at+sEPfqBJkyapsLBQDQ0NSkpK0vbt2zVr1ixJUmVlpcaNG6dQKKTp06dr7969evjhh1VdXa3k5GRJUlFRkfLz81VXVyeXy6X8/HyVlJTo5MmTzvecM2eO6uvrVVpaKknKyMjQ1KlTtXnzZklSW1ubUlNTtWTJEq1YsaJLu3QmEonI6/WqoaFBHo+nu7+m29qoFSWxXgE96L11WbFeAT2I57ctFp/f3fnz+7peEcrNzVVWVpYyMzOj7i8vL1dLS0vU/WPHjtXIkSMVCoUkSaFQSOPHj3ciSJICgYAikYhOnTrlzHz03IFAwDlHc3OzysvLo2bi4+OVmZnpzHRll49qampSJBKJugEAgN6rT3cf8Otf/1rHjx/X0aNHrzgWDoflcrk0ePDgqPuTk5MVDoedmcsjqON4x7FrzUQiEX344Yf64IMP1NraetWZysrKLu/yUWvXrtX3vve9a/z0AACgN+nWK0Jnz57VM888o5deekmJiYk3a6eYWblypRoaGpzb2bNnY70SAAC4iboVQuXl5aqtrdXkyZPVp08f9enTR4cOHdKmTZvUp08fJScnq7m5WfX19VGPq6mpkc/nkyT5fL4rPrnV8XVnMx6PR/369dOwYcOUkJBw1ZnLz9HZLh/ldrvl8XiibgAAoPfqVgg98MADOnHihCoqKpzblClTNHfuXOe/+/btq7KyMucxp0+fVlVVlfx+vyTJ7/frxIkTUZ/u2r9/vzwej9LS0pyZy8/RMdNxDpfLpfT09KiZtrY2lZWVOTPp6emd7gIAAGzr1nuEBg0apHvuuSfqvgEDBmjo0KHO/QsWLFAwGNSQIUPk8Xi0ZMkS+f1+51NaM2bMUFpamh5//HGtX79e4XBYq1atUm5urtxutyRp4cKF2rx5s5YvX6758+frwIED2rlzp0pK/vtJh2AwqJycHE2ZMkXTpk1TYWGhGhsbNW/ePEmS1+vtdBcAAGBbt98s3ZkNGzYoPj5e2dnZampqUiAQ0NatW53jCQkJ2rNnjxYtWiS/368BAwYoJydHa9ascWZGjx6tkpIS5eXlaePGjRoxYoReeOEFBQIBZ2b27Nmqq6tTQUGBwuGwJk2apNLS0qg3UHe2CwAAsO26riNkBdcRghUWrzNiGc9vWyw+v2/6dYQAAAB6A0IIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzuhVC27Zt04QJE+TxeOTxeOT3+7V3717n+MWLF5Wbm6uhQ4dq4MCBys7OVk1NTdQ5qqqqlJWVpf79+2v48OFatmyZLl26FDVz8OBBTZ48WW63W2PGjFFxcfEVu2zZskWjRo1SYmKiMjIydOTIkajjXdkFAADY1q0QGjFihNatW6fy8nIdO3ZMX/rSl/TVr35Vp06dkiTl5eXp1Vdf1a5du3To0CFVV1fr0UcfdR7f2tqqrKwsNTc36/Dhw3rxxRdVXFysgoICZ+bMmTPKysrS/fffr4qKCi1dulRPPfWU9u3b58zs2LFDwWBQq1ev1vHjxzVx4kQFAgHV1tY6M53tAgAAENfe3t5+IycYMmSInn/+ec2aNUtJSUnavn27Zs2aJUmqrKzUuHHjFAqFNH36dO3du1cPP/ywqqurlZycLEkqKipSfn6+6urq5HK5lJ+fr5KSEp08edL5HnPmzFF9fb1KS0slSRkZGZo6dao2b94sSWpra1NqaqqWLFmiFStWqKGhodNduiISicjr9aqhoUEej+dGfk23nVErSmK9AnrQe+uyYr0CehDPb1ssPr+78+f3db9HqLW1Vb/+9a/V2Ngov9+v8vJytbS0KDMz05kZO3asRo4cqVAoJEkKhUIaP368E0GSFAgEFIlEnFeVQqFQ1Dk6ZjrO0dzcrPLy8qiZ+Ph4ZWZmOjNd2eVqmpqaFIlEom4AAKD36nYInThxQgMHDpTb7dbChQv18ssvKy0tTeFwWC6XS4MHD46aT05OVjgcliSFw+GoCOo43nHsWjORSEQffvihzp07p9bW1qvOXH6Ozna5mrVr18rr9Tq31NTUrv1SAADAbanbIXT33XeroqJCb731lhYtWqScnBz9+c9/vhm79biVK1eqoaHBuZ09ezbWKwEAgJuoT3cf4HK5NGbMGElSenq6jh49qo0bN2r27Nlqbm5WfX191CsxNTU18vl8kiSfz3fFp7s6Psl1+cxHP91VU1Mjj8ejfv36KSEhQQkJCVedufwcne1yNW63W263uxu/DQAAcDu74esItbW1qampSenp6erbt6/KysqcY6dPn1ZVVZX8fr8kye/368SJE1Gf7tq/f788Ho/S0tKcmcvP0THTcQ6Xy6X09PSomba2NpWVlTkzXdkFAACgW68IrVy5Ug899JBGjhyp8+fPa/v27Tp48KD27dsnr9erBQsWKBgMasiQIfJ4PFqyZIn8fr/zKa0ZM2YoLS1Njz/+uNavX69wOKxVq1YpNzfXeSVm4cKF2rx5s5YvX6758+frwIED2rlzp0pK/vsph2AwqJycHE2ZMkXTpk1TYWGhGhsbNW/ePEnq0i4AAADdCqHa2lo98cQT+te//iWv16sJEyZo3759+vKXvyxJ2rBhg+Lj45Wdna2mpiYFAgFt3brVeXxCQoL27NmjRYsWye/3a8CAAcrJydGaNWucmdGjR6ukpER5eXnauHGjRowYoRdeeEGBQMCZmT17turq6lRQUKBwOKxJkyaptLQ06g3Une0CAABww9cR6s24jhCssHidEct4ftti8fndI9cRAgAAuN0RQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmNWtEFq7dq2mTp2qQYMGafjw4Zo5c6ZOnz4dNXPx4kXl5uZq6NChGjhwoLKzs1VTUxM1U1VVpaysLPXv31/Dhw/XsmXLdOnSpaiZgwcPavLkyXK73RozZoyKi4uv2GfLli0aNWqUEhMTlZGRoSNHjnR7FwAAYFe3QujQoUPKzc3Vm2++qf3796ulpUUzZsxQY2OjM5OXl6dXX31Vu3bt0qFDh1RdXa1HH33UOd7a2qqsrCw1Nzfr8OHDevHFF1VcXKyCggJn5syZM8rKytL999+viooKLV26VE899ZT27dvnzOzYsUPBYFCrV6/W8ePHNXHiRAUCAdXW1nZ5FwAAYFtce3t7+/U+uK6uTsOHD9ehQ4d03333qaGhQUlJSdq+fbtmzZolSaqsrNS4ceMUCoU0ffp07d27Vw8//LCqq6uVnJwsSSoqKlJ+fr7q6urkcrmUn5+vkpISnTx50vlec+bMUX19vUpLSyVJGRkZmjp1qjZv3ixJamtrU2pqqpYsWaIVK1Z0aZePampqUlNTk/N1JBJRamqqGhoa5PF4rvfXdFsataIk1iugB723LivWK6AH8fy2xeLzOxKJyOv1dunP7xt6j1BDQ4MkaciQIZKk8vJytbS0KDMz05kZO3asRo4cqVAoJEkKhUIaP368E0GSFAgEFIlEdOrUKWfm8nN0zHSco7m5WeXl5VEz8fHxyszMdGa6sstHrV27Vl6v17mlpqZe3y8GAADcFq47hNra2rR06VJ97nOf0z333CNJCofDcrlcGjx4cNRscnKywuGwM3N5BHUc7zh2rZlIJKIPP/xQ586dU2tr61VnLj9HZ7t81MqVK9XQ0ODczp4928XfBgAAuB31ud4H5ubm6uTJk3r99dc/zn1iyu12y+12x3oNAADQQ67rFaHFixdrz549+sMf/qARI0Y49/t8PjU3N6u+vj5qvqamRj6fz5n56Ce3Or7ubMbj8ahfv34aNmyYEhISrjpz+Tk62wUAANjWrRBqb2/X4sWL9fLLL+vAgQMaPXp01PH09HT17dtXZWVlzn2nT59WVVWV/H6/JMnv9+vEiRNRn+7av3+/PB6P0tLSnJnLz9Ex03EOl8ul9PT0qJm2tjaVlZU5M13ZBQAA2NatvxrLzc3V9u3b9Zvf/EaDBg1y3mvj9XrVr18/eb1eLViwQMFgUEOGDJHH49GSJUvk9/udT2nNmDFDaWlpevzxx7V+/XqFw2GtWrVKubm5zl9LLVy4UJs3b9by5cs1f/58HThwQDt37lRJyX8/6RAMBpWTk6MpU6Zo2rRpKiwsVGNjo+bNm+fs1NkuAADAtm6F0LZt2yRJX/ziF6Pu/+Uvf6knn3xSkrRhwwbFx8crOztbTU1NCgQC2rp1qzObkJCgPXv2aNGiRfL7/RowYIBycnK0Zs0aZ2b06NEqKSlRXl6eNm7cqBEjRuiFF15QIBBwZmbPnq26ujoVFBQoHA5r0qRJKi0tjXoDdWe7AAAA227oOkK9XXeuQ9DbcJ0RWyxeZ8Qynt+2WHx+99h1hAAAAG5nhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCr2yH02muv6ZFHHlFKSori4uK0e/fuqOPt7e0qKCjQHXfcoX79+ikzM1Pvvvtu1Mz777+vuXPnyuPxaPDgwVqwYIEuXLgQNfPOO+/o3nvvVWJiolJTU7V+/fordtm1a5fGjh2rxMREjR8/Xr/97W+7vQsAALCr2yHU2NioiRMnasuWLVc9vn79em3atElFRUV66623NGDAAAUCAV28eNGZmTt3rk6dOqX9+/drz549eu211/T00087xyORiGbMmKE777xT5eXlev755/Xcc8/pZz/7mTNz+PBhPfbYY1qwYIH+9Kc/aebMmZo5c6ZOnjzZrV0AAIBdce3t7e3X/eC4OL388suaOXOmpP+8ApOSkqJvf/vbevbZZyVJDQ0NSk5OVnFxsebMmaO//OUvSktL09GjRzVlyhRJUmlpqb7yla/on//8p1JSUrRt2zZ997vfVTgclsvlkiStWLFCu3fvVmVlpSRp9uzZamxs1J49e5x9pk+frkmTJqmoqKhLu3QmEonI6/WqoaFBHo/nen9Nt6VRK0pivQJ60HvrsmK9AnoQz29bLD6/u/Pn98f6HqEzZ84oHA4rMzPTuc/r9SojI0OhUEiSFAqFNHjwYCeCJCkzM1Px8fF66623nJn77rvPiSBJCgQCOn36tD744ANn5vLv0zHT8X26sstHNTU1KRKJRN0AAEDv9bGGUDgcliQlJydH3Z+cnOwcC4fDGj58eNTxPn36aMiQIVEzVzvH5d/jf81cfryzXT5q7dq18nq9zi01NbULPzUAALhd8amxy6xcuVINDQ3O7ezZs7FeCQAA3EQfawj5fD5JUk1NTdT9NTU1zjGfz6fa2tqo45cuXdL7778fNXO1c1z+Pf7XzOXHO9vlo9xutzweT9QNAAD0Xh9rCI0ePVo+n09lZWXOfZFIRG+99Zb8fr8kye/3q76+XuXl5c7MgQMH1NbWpoyMDGfmtddeU0tLizOzf/9+3X333frEJz7hzFz+fTpmOr5PV3YBAAC2dTuELly4oIqKClVUVEj6z5uSKyoqVFVVpbi4OC1dulQ/+MEP9Morr+jEiRN64oknlJKS4nyybNy4cXrwwQf1jW98Q0eOHNEbb7yhxYsXa86cOUpJSZEkff3rX5fL5dKCBQt06tQp7dixQxs3blQwGHT2eOaZZ1RaWqof/ehHqqys1HPPPadjx45p8eLFktSlXQAAgG19uvuAY8eO6f7773e+7oiTnJwcFRcXa/ny5WpsbNTTTz+t+vp6ff7zn1dpaakSExOdx7z00ktavHixHnjgAcXHxys7O1ubNm1yjnu9Xv3ud79Tbm6u0tPTNWzYMBUUFERda+izn/2stm/frlWrVuk73/mO7rrrLu3evVv33HOPM9OVXQAAgF03dB2h3o7rCMEKi9cZsYznty0Wn98xu44QAADA7YQQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALEIIAACYRQgBAACzCCEAAGAWIQQAAMwihAAAgFmEEAAAMIsQAgAAZhFCAADALBMhtGXLFo0aNUqJiYnKyMjQkSNHYr0SAAC4BfT6ENqxY4eCwaBWr16t48ePa+LEiQoEAqqtrY31agAAIMZ6fQj9+Mc/1je+8Q3NmzdPaWlpKioqUv/+/fWLX/wi1qsBAIAY6xPrBW6m5uZmlZeXa+XKlc598fHxyszMVCgUumK+qalJTU1NztcNDQ2SpEgkcvOXvcW0Nf2/WK+AHmTxf+OW8fy2xeLzu+Nnbm9v73S2V4fQuXPn1NraquTk5Kj7k5OTVVlZecX82rVr9b3vfe+K+1NTU2/ajsCtwFsY6w0A3CyWn9/nz5+X1+u95kyvDqHuWrlypYLBoPN1W1ub3n//fQ0dOlRxcXEx3Aw9IRKJKDU1VWfPnpXH44n1OgA+Rjy/bWlvb9f58+eVkpLS6WyvDqFhw4YpISFBNTU1UffX1NTI5/NdMe92u+V2u6PuGzx48M1cEbcgj8fD/1ECvRTPbzs6eyWoQ69+s7TL5VJ6errKysqc+9ra2lRWVia/3x/DzQAAwK2gV78iJEnBYFA5OTmaMmWKpk2bpsLCQjU2NmrevHmxXg0AAMRYrw+h2bNnq66uTgUFBQqHw5o0aZJKS0uveAM14Ha7tXr16iv+ehTA7Y/nN/6XuPaufLYMAACgF+rV7xECAAC4FkIIAACYRQgBAACzCCEAAGAWIQQAAMzq9R+fBwDYc+7cOf3iF79QKBRSOByWJPl8Pn32s5/Vk08+qaSkpBhviFsFrwgB/8PZs2c1f/78WK8BoJuOHj2qT33qU9q0aZO8Xq/uu+8+3XffffJ6vdq0aZPGjh2rY8eOxXpN3CK4jhDwP7z99tuaPHmyWltbY70KgG6YPn26Jk6cqKKioiv+wez29nYtXLhQ77zzjkKhUIw2xK2EvxqDWa+88so1j//jH//ooU0AfJzefvttFRcXXxFBkhQXF6e8vDx95jOficFmuBURQjBr5syZiouL07VeFL3a/5ECuLX5fD4dOXJEY8eOverxI0eO8M8swUEIwaw77rhDW7du1Ve/+tWrHq+oqFB6enoPbwXgRj377LN6+umnVV5ergceeMCJnpqaGpWVlennP/+5fvjDH8Z4S9wqCCGYlZ6ervLy8v8ZQp29WgTg1pSbm6thw4Zpw4YN2rp1q/M+v4SEBKWnp6u4uFhf+9rXYrwlbhW8WRpm/fGPf1RjY6MefPDBqx5vbGzUsWPH9IUvfKGHNwPwcWlpadG5c+ckScOGDVPfvn1jvBFuNYQQAAAwi+sIAQAAswghAABgFiEEAADMIoQAAIBZhBCA29oXv/hFLV26tEuzBw8eVFxcnOrr62/oe44aNUqFhYU3dA4AtwZCCAAAmEUIAQAAswghAL3Gr371K02ZMkWDBg2Sz+fT17/+ddXW1l4x98Ybb2jChAlKTEzU9OnTdfLkyajjr7/+uu69917169dPqamp+ta3vqXGxsae+jEA9CBCCECv0dLSou9///t6++23tXv3br333nt68sknr5hbtmyZfvSjH+no0aNKSkrSI488opaWFknS3//+dz344IPKzs7WO++8ox07duj111/X4sWLe/inAdAT+LfGAPQa8+fPd/77k5/8pDZt2qSpU6fqwoULGjhwoHNs9erV+vKXvyxJevHFFzVixAi9/PLL+trXvqa1a9dq7ty5zhuw77rrLm3atElf+MIXtG3bNiUmJvbozwTg5uIVIQC9Rnl5uR555BGNHDlSgwYNcv6duKqqqqg5v9/v/PeQIUN099136y9/+Ysk6e2331ZxcbEGDhzo3AKBgNra2nTmzJme+2EA9AheEQLQKzQ2NioQCCgQCOill15SUlKSqqqqFAgE1Nzc3OXzXLhwQd/85jf1rW9964pjI0eO/DhXBnALIIQA9AqVlZX697//rXXr1ik1NVWSdOzYsavOvvnmm07UfPDBB/rrX/+qcePGSZImT56sP//5zxozZkzPLA4gpvirMQC9wsiRI+VyufSTn/xE//jHP/TKK6/o+9///lVn16xZo7KyMp08eVJPPvmkhg0bppkzZ0qS8vPzdfjwYS1evFgVFRV699139Zvf/IY3SwO9FCEEoFdISkpScXGxdu3apbS0NK1bt04//OEPrzq7bt06PfPMM0pPT1c4HNarr74ql8slSZowYYIOHTqkv/71r7r33nv1mc98RgUFBUpJSenJHwdAD4lrb29vj/USAAAAscArQgAAwCxCCAAAmEUIAQAAswghAABgFiEEAADMIoQAAIBZhBAAADCLEAIAAGYRQgAAwCxCCAAAmEUIAQAAs/4/owpHejtr+e0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sarcasm_df[\"label\"].value_counts().plot(kind=\"bar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build JSON Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_to_name(label):\n",
    "    if label == 1:\n",
    "        return \"Sarcastic\"\n",
    "    return \"Not Sarcastic\"\n",
    "\n",
    "\n",
    "dataset_data = [\n",
    "    {\n",
    "        \"instruction\": \"Detect if the comment is sarcastic or not.\",\n",
    "        \"input\": row_dict[\"comment\"],\n",
    "        \"output\": label_to_name(row_dict[\"label\"]),\n",
    "    }\n",
    "    for row_dict in sarcasm_df.to_dict(orient=\"records\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'instruction': 'Detect if the comment is sarcastic or not.',\n",
       " 'input': 'NC and NH.',\n",
       " 'output': 'Not Sarcastic'}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"sarcasm-dataset-alpaca.json\", \"w\") as f:\n",
    "    json.dump(dataset_data, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 33/33 [00:15<00:00,  2.08it/s]\n",
      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
      "The tokenizer class you load from this checkpoint is 'LLaMATokenizer'. \n",
      "The class this function is called from is 'LlamaTokenizer'.\n"
     ]
    }
   ],
   "source": [
    "BASE_MODEL = \"decapoda-research/llama-7b-hf\"\n",
    "\n",
    "model = LlamaForCausalLM.from_pretrained(\n",
    "    BASE_MODEL,\n",
    "    load_in_8bit=True,\n",
    "    torch_dtype=torch.float16,\n",
    "    device_map=\"balanced\",\n",
    ")\n",
    "\n",
    "tokenizer = LlamaTokenizer.from_pretrained(BASE_MODEL)\n",
    "\n",
    "tokenizer.pad_token_id = 0  # unk. we want this to be different from the eos token\n",
    "tokenizer.padding_side = \"left\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset json (/home/user/ishfaqm0/.cache/huggingface/datasets/json/default-d18af4156e32a0a3/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\n",
      "100%|██████████| 1/1 [00:00<00:00, 73.70it/s]\n"
     ]
    }
   ],
   "source": [
    "data = load_dataset(\"json\", data_files=\"sarcasm-dataset-alpaca.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['instruction', 'output', 'input'],\n",
       "    num_rows: 1007783\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "CUTOFF_LEN = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_prompt(data_point):\n",
    "    return f\"\"\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.  # noqa: E501\n",
    "### Instruction:\n",
    "{data_point[\"instruction\"]}\n",
    "### Input:\n",
    "{data_point[\"input\"]}\n",
    "### Response:\n",
    "{data_point[\"output\"]}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(prompt, add_eos_token=True):\n",
    "    result = tokenizer(\n",
    "        prompt,\n",
    "        truncation=True,\n",
    "        max_length=CUTOFF_LEN,\n",
    "        padding=False,\n",
    "        return_tensors=None,\n",
    "    )\n",
    "    if (\n",
    "        result[\"input_ids\"][-1] != tokenizer.eos_token_id\n",
    "        and len(result[\"input_ids\"]) < CUTOFF_LEN\n",
    "        and add_eos_token\n",
    "    ):\n",
    "        result[\"input_ids\"].append(tokenizer.eos_token_id)\n",
    "        result[\"attention_mask\"].append(1)\n",
    "\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "\n",
    "    return result\n",
    "\n",
    "\n",
    "def generate_and_tokenize_prompt(data_point):\n",
    "    full_prompt = generate_prompt(data_point)\n",
    "    tokenized_full_prompt = tokenize(full_prompt)\n",
    "    return tokenized_full_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached split indices for dataset at /home/user/ishfaqm0/.cache/huggingface/datasets/json/default-d18af4156e32a0a3/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-a5b94b5b4dfc51ec.arrow and /home/user/ishfaqm0/.cache/huggingface/datasets/json/default-d18af4156e32a0a3/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e6ac80fcf5a4bbe7.arrow\n",
      "Loading cached shuffled indices for dataset at /home/user/ishfaqm0/.cache/huggingface/datasets/json/default-d18af4156e32a0a3/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-ced8f71d4e2ad9f0.arrow\n",
      "Loading cached processed dataset at /home/user/ishfaqm0/.cache/huggingface/datasets/json/default-d18af4156e32a0a3/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e932a45aa204c7bd.arrow\n",
      "Loading cached shuffled indices for dataset at /home/user/ishfaqm0/.cache/huggingface/datasets/json/default-d18af4156e32a0a3/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-92da027d1a44b991.arrow\n",
      "Loading cached processed dataset at /home/user/ishfaqm0/.cache/huggingface/datasets/json/default-d18af4156e32a0a3/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-eec5317efdd4c733.arrow\n"
     ]
    }
   ],
   "source": [
    "train_val = data[\"train\"].train_test_split(test_size=0.25, shuffle=True, seed=42)\n",
    "train_data = train_val[\"train\"].shuffle().map(generate_and_tokenize_prompt)\n",
    "val_data = train_val[\"test\"].shuffle().map(generate_and_tokenize_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                   \r"
     ]
    }
   ],
   "source": [
    "# save train and val data\n",
    "train_data.save_to_disk(\"train_data\")\n",
    "val_data.save_to_disk(\"val_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mfaizi-ch\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.4"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/user/ishfaqm0/Sarcasm-Detection/wandb/run-20230704_201020-7g0o6zez</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/faizi-ch/Sarcasm-Detection/runs/7g0o6zez' target=\"_blank\">alpaca-128-batch-no-gradient-accumulation-batch-64</a></strong> to <a href='https://wandb.ai/faizi-ch/Sarcasm-Detection' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/faizi-ch/Sarcasm-Detection' target=\"_blank\">https://wandb.ai/faizi-ch/Sarcasm-Detection</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/faizi-ch/Sarcasm-Detection/runs/7g0o6zez' target=\"_blank\">https://wandb.ai/faizi-ch/Sarcasm-Detection/runs/7g0o6zez</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/faizi-ch/Sarcasm-Detection/runs/7g0o6zez?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7f00500cf160>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.init(\n",
    "    project=\"sarcasm-detection\",\n",
    "    name=\"alpaca-128-batch-no-gradient-accumulation-batch-64\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: WANDB_PROJECT=Sarcasm-Detection\n"
     ]
    }
   ],
   "source": [
    "%env WANDB_PROJECT=Sarcasm-Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "LORA_R = 8\n",
    "LORA_ALPHA = 16\n",
    "LORA_DROPOUT = 0.05\n",
    "LORA_TARGET_MODULES = [\n",
    "    \"q_proj\",\n",
    "    \"v_proj\",\n",
    "]\n",
    "\n",
    "BATCH_SIZE = 1024\n",
    "MICRO_BATCH_SIZE = 64\n",
    "GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE\n",
    "LEARNING_RATE = 3e-4\n",
    "TRAIN_STEPS = 300\n",
    "OUTPUT_DIR = \"experiments\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/peft/utils/other.py:102: FutureWarning: prepare_model_for_int8_training is deprecated and will be removed in a future version. Use prepare_model_for_kbit_training instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 4,194,304 || all params: 6,742,609,920 || trainable%: 0.06220594176090199\n"
     ]
    }
   ],
   "source": [
    "model = prepare_model_for_int8_training(model)\n",
    "config = LoraConfig(\n",
    "    r=LORA_R,\n",
    "    lora_alpha=LORA_ALPHA,\n",
    "    target_modules=LORA_TARGET_MODULES,\n",
    "    lora_dropout=LORA_DROPOUT,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "model = get_peft_model(model, config)\n",
    "model.print_trainable_parameters()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_arguments = transformers.TrainingArguments(\n",
    "    per_device_train_batch_size=MICRO_BATCH_SIZE,\n",
    "    # gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,\n",
    "    warmup_steps=100,\n",
    "    # max_steps=TRAIN_STEPS,\n",
    "    num_train_epochs=1,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    fp16=True,\n",
    "    logging_steps=10,\n",
    "    optim=\"adamw_torch\",\n",
    "    evaluation_strategy=\"steps\",\n",
    "    save_strategy=\"steps\",\n",
    "    eval_steps=500,\n",
    "    save_steps=500,\n",
    "    output_dir=OUTPUT_DIR,\n",
    "    # save_total_limit=3,\n",
    "    load_best_model_at_end=True,\n",
    "    report_to=\"wandb\",\n",
    "    run_name=\"alpaca-128-batch-no-gradient-accumulation-batch-64\",\n",
    "    dataloader_num_workers=16,\n",
    "    seed=42,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = transformers.DataCollatorForSeq2Seq(\n",
    "    tokenizer, pad_to_multiple_of=8, return_tensors=\"pt\", padding=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:318: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1559' max='11810' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 1559/11810 16:31:17 < 108:46:28, 0.03 it/s, Epoch 0.13/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.661400</td>\n",
       "      <td>0.637272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>2.127300</td>\n",
       "      <td>2.336339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>2.128100</td>\n",
       "      <td>2.336339</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:318: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:318: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/home/user/ishfaqm0/miniconda3/envs/nlp2/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:318: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n"
     ]
    }
   ],
   "source": [
    "trainer = transformers.Trainer(\n",
    "    model=model,\n",
    "    train_dataset=train_data,\n",
    "    eval_dataset=val_data,\n",
    "    args=training_arguments,\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "model.config.use_cache = False\n",
    "old_state_dict = model.state_dict\n",
    "model.state_dict = (\n",
    "    lambda self, *_, **__: get_peft_model_state_dict(self, old_state_dict())\n",
    ").__get__(model, type(model))\n",
    "\n",
    "model = torch.compile(model)\n",
    "\n",
    "trainer.train()\n",
    "model.save_pretrained(OUTPUT_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}