drs_text_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 28215 datasets currently available on the hub\n",
      "The first 10 are: ['acronym_identification', 'ade_corpus_v2', 'adversarial_qa', 'aeslc', 'afrikaans_ner_corpus', 'ag_news', 'ai2_arc', 'air_dialogue', 'ajgt_twitter_ar', 'allegro_reviews']\n"
     ]
    }
   ],
   "source": [
    "from datasets import list_datasets\n",
    "\n",
    "all_datasets = list_datasets()\n",
    "print(f\"There are {len(all_datasets)} datasets currently available on the hub\")\n",
    "print(f\"The first 10 are: {all_datasets[:10]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No config specified, defaulting to: emotion/split\n",
      "Found cached dataset emotion (C:/Users/victor/.cache/huggingface/datasets/emotion/split/1.0.0/cca5efe2dfeb58c1d098e0f9eeb200e9927d889b5a03c67097275dfb5fe463bd)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8a8e899ee464d71a4ead033ffa6fc9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "emotions = load_dataset(\"emotion\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 16000\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emotions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label'],\n",
       "    num_rows: 16000\n",
       "})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds = emotions[\"train\"]\n",
    "train_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i didnt feel humiliated</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i can go from feeling so hopeless to so damned...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>im grabbing a minute to post i feel greedy wrong</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i am ever feeling nostalgic about the fireplac...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i am feeling grouchy</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label\n",
       "0                            i didnt feel humiliated      0\n",
       "1  i can go from feeling so hopeless to so damned...      0\n",
       "2   im grabbing a minute to post i feel greedy wrong      3\n",
       "3  i am ever feeling nostalgic about the fireplac...      2\n",
       "4                               i am feeling grouchy      3"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "emotions.set_format(type=\"pandas\")\n",
    "df = emotions[\"train\"][:]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i didnt feel humiliated</td>\n",
       "      <td>0</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i can go from feeling so hopeless to so damned...</td>\n",
       "      <td>0</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>im grabbing a minute to post i feel greedy wrong</td>\n",
       "      <td>3</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i am ever feeling nostalgic about the fireplac...</td>\n",
       "      <td>2</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i am feeling grouchy</td>\n",
       "      <td>3</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label label_name\n",
       "0                            i didnt feel humiliated      0    sadness\n",
       "1  i can go from feeling so hopeless to so damned...      0    sadness\n",
       "2   im grabbing a minute to post i feel greedy wrong      3      anger\n",
       "3  i am ever feeling nostalgic about the fireplac...      2       love\n",
       "4                               i am feeling grouchy      3      anger"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def label_int2str(row):\n",
    "    return emotions[\"train\"].features[\"label\"].int2str(row)\n",
    "\n",
    "df[\"label_name\"] = df[\"label\"].apply(label_int2str)\n",
    "df.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformers Features"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using pretrained models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'Automodel' from 'transformers' (C:\\Users\\victor\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\transformers\\__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[19], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtransformers\u001b[39;00m \u001b[39mimport\u001b[39;00m Automodel\n\u001b[0;32m      3\u001b[0m model_ckpt \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mdistilbert-base-uncased\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m      4\u001b[0m device \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mdevice(\u001b[39m\"\u001b[39m\u001b[39mcuda\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39mcuda\u001b[39m.\u001b[39mis_available() \u001b[39melse\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[1;31mImportError\u001b[0m: cannot import name 'Automodel' from 'transformers' (C:\\Users\\victor\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\transformers\\__init__.py)"
     ]
    }
   ],
   "source": [
    "from transformers import Automodel\n",
    "\n",
    "model_ckpt = \"distilbert-base-uncased\"\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = Automodel.from_pretrained(model_ckpt).to(device)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extracting  the last hidden states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"this is a test\"\n",
    "inputs = tokenizer(text, return_tensors=\"pt\")\n",
    "print(f\"Input tensor shape: {inputs['input_ids'],size()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = {k:v.to(device) for kv, in inputs.items()}\n",
    "with torch.no_grad():\n",
    "    outputs = model(**inputs)\n",
    "print(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs.last_hidden_state.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs.last_hidden_state[:,0].size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_hidden_states(batch):\n",
    "    # Place model inputs on the GPU\n",
    "    inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}\n",
    "    # Extract last hidden states\n",
    "    with torch.no_grad():\n",
    "        last_hidden_state = model(**inputs).last_hidden_state\n",
    "    # Return vector for [CLS] token\n",
    "    return {\"hidden_state\": last_hidden_state[:,0].cpu().numpy()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions_encoded.set_format(\"torch\",\n",
    "                            columns=[\"inputs_ids\", \"attention_mask\", \"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions_hidden = emotions_encoded.map(extract_hidden_states, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions_hidden[\"train\"].column_names"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a feature matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "X_train = np.array(emotions_hidden[\"train\"][\"hidden_state\"])\n",
    "X_valid = np.array(emotions_hidden[\"validation\"][\"hidden_state\"])\n",
    "y_train = np.array(emotions_hidden[\"train\"][\"label\"])\n",
    "y_valid = np.array(emotions_hidden[\"validation\"][\"label\"])\n",
    "X_train.shape, X_valid.shape"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualizing the training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from umap import UMAP\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import pandas as pd\n",
    "\n",
    "# Scale features [0,1] range\n",
    "X_scaled = MinMaxScaler().fit_transform(X_train)\n",
    "# Initialize and fit UMAP\n",
    "mapper = UMAP(n_components=2, metric=\"cosine\").fit(X_scaled)\n",
    "# Create a Dataframe of 2D embeddings\n",
    "df_emb = pd.DataFrame(mapper.embedding_, column=[\"X\", \"Y\"])\n",
    "df_emb[\"emb\"] = y_train\n",
    "df_emb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 3, figsize(7,5))\n",
    "axes = axes.flatten()\n",
    "cmaps = [\"Greys\", \"Blues\", \"Oranges\", \"Reds\", \"Purples\", \"Greens\"]\n",
    "labels = emotions[\"train\"].features[\"label\"].names\n",
    "\n",
    "for i, (label, cmap) in enumerate(zip(labels, cmaps)):\n",
    "    df_emb_sub = df_emb.query(f\"label =={i}\")\n",
    "    axes[i].hexbin(df_emb_sub[\"X\"], df_emb_sub[\"Y\"], cmap=cmap, gridsize=20, linewidths=(0,))\n",
    "    axes[i].set_title(label)\n",
    "    axes[i].set_xticks([]), axes[i].set_yticks([])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training a simple classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-tuning transformers"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}