2-Template-fastText.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import fasttext\n",
    "import gensim\n",
    "import sklearn.metrics\n",
    "import re\n",
    "import unicodedata\n",
    "from tqdm import tqdm\n",
    "import sys\n",
    "import hashlib\n",
    "import random\n",
    "\n",
    "train_file = 'nlbse23-issue-classification-train.csv'\n",
    "test_file = 'nlbse23-issue-classification-test.csv'\n",
    "\n",
    "csv.field_size_limit(sys.maxsize) # to avoid error: _csv.Error: field larger than field limit (131072)\n",
    "\n",
    "def count_csv_rows(filename):\n",
    "\twith open(filename, \"r\", newline='', encoding='utf-8') as f:\n",
    "\t\treturn sum(1 for _ in csv.DictReader(f))\n",
    "\n",
    "def print_csv_preview(filename):\n",
    "\tprint(filename)\n",
    "\tprint(\"total rows\", count_csv_rows(filename))\n",
    "\tdisplay(pd.read_csv(filename, nrows=5))\n",
    "\t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nlbse23-issue-classification-train.csv\n",
      "total rows 1275881\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>labels</th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>author_association</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1199051804</td>\n",
       "      <td>documentation</td>\n",
       "      <td>setting a logging Handler name</td>\n",
       "      <td>BPO | [43058](https://bugs.python.org/issue430...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1199074324</td>\n",
       "      <td>documentation</td>\n",
       "      <td>Improve documentation for typing._GenericAlias</td>\n",
       "      <td>BPO | [46589](https://bugs.python.org/issue465...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1199022454</td>\n",
       "      <td>documentation</td>\n",
       "      <td>Description of '\\w' behavior is vague in `re` ...</td>\n",
       "      <td>BPO | [38566](https://bugs.python.org/issue385...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1199028356</td>\n",
       "      <td>documentation</td>\n",
       "      <td>add docstrings to functions in pdb module</td>\n",
       "      <td>BPO | [39278](https://bugs.python.org/issue392...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1199055394</td>\n",
       "      <td>documentation</td>\n",
       "      <td>Documentation needs to declare CalledProcessEr...</td>\n",
       "      <td>BPO | [43635](https://bugs.python.org/issue436...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id         labels  \\\n",
       "0  1199051804  documentation   \n",
       "1  1199074324  documentation   \n",
       "2  1199022454  documentation   \n",
       "3  1199028356  documentation   \n",
       "4  1199055394  documentation   \n",
       "\n",
       "                                               title  \\\n",
       "0                     setting a logging Handler name   \n",
       "1     Improve documentation for typing._GenericAlias   \n",
       "2  Description of '\\w' behavior is vague in `re` ...   \n",
       "3          add docstrings to functions in pdb module   \n",
       "4  Documentation needs to declare CalledProcessEr...   \n",
       "\n",
       "                                                body author_association  \n",
       "0  BPO | [43058](https://bugs.python.org/issue430...          MANNEQUIN  \n",
       "1  BPO | [46589](https://bugs.python.org/issue465...          MANNEQUIN  \n",
       "2  BPO | [38566](https://bugs.python.org/issue385...          MANNEQUIN  \n",
       "3  BPO | [39278](https://bugs.python.org/issue392...          MANNEQUIN  \n",
       "4  BPO | [43635](https://bugs.python.org/issue436...          MANNEQUIN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nlbse23-issue-classification-test.csv\n",
      "total rows 142320\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>labels</th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>author_association</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1199053386</td>\n",
       "      <td>documentation</td>\n",
       "      <td>A possible misleading expression in the Virtua...</td>\n",
       "      <td>BPO | [43319](https://bugs.python.org/issue433...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1255069635</td>\n",
       "      <td>bug</td>\n",
       "      <td>[BUG] a valid `gameName` in the `create a new ...</td>\n",
       "      <td>**Describe the bug**\\r\\nIn the `create a new p...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1089772715</td>\n",
       "      <td>feature</td>\n",
       "      <td>How to check if a certain entity still exists?</td>\n",
       "      <td>During a bug in my own code I noticed that the...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000928729</td>\n",
       "      <td>feature</td>\n",
       "      <td>chose the timezone in dbeaver option</td>\n",
       "      <td>Dbeaver 21.2.0\\r\\n\\r\\nFor all version DBeaver,...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1300011093</td>\n",
       "      <td>bug</td>\n",
       "      <td>[Issue]: Multiple Versions of a Movie not work...</td>\n",
       "      <td>### Please describe your bug\\n\\nThe doc at htt...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id         labels  \\\n",
       "0  1199053386  documentation   \n",
       "1  1255069635            bug   \n",
       "2  1089772715        feature   \n",
       "3  1000928729        feature   \n",
       "4  1300011093            bug   \n",
       "\n",
       "                                               title  \\\n",
       "0  A possible misleading expression in the Virtua...   \n",
       "1  [BUG] a valid `gameName` in the `create a new ...   \n",
       "2     How to check if a certain entity still exists?   \n",
       "3               chose the timezone in dbeaver option   \n",
       "4  [Issue]: Multiple Versions of a Movie not work...   \n",
       "\n",
       "                                                body author_association  \n",
       "0  BPO | [43319](https://bugs.python.org/issue433...          MANNEQUIN  \n",
       "1  **Describe the bug**\\r\\nIn the `create a new p...               NONE  \n",
       "2  During a bug in my own code I noticed that the...               NONE  \n",
       "3  Dbeaver 21.2.0\\r\\n\\r\\nFor all version DBeaver,...               NONE  \n",
       "4  ### Please describe your bug\\n\\nThe doc at htt...               NONE  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# download the training set if it does not exist\n",
    "if not os.path.isfile(train_file):\n",
    "  !curl \"https://tickettagger.blob.core.windows.net/datasets/{train_file}.tar.gz\" | tar -xz\n",
    "\n",
    "print_csv_preview(train_file)\n",
    "\n",
    "if not os.path.isfile(test_file):\n",
    "  !curl \"https://tickettagger.blob.core.windows.net/datasets/{test_file}.tar.gz\" | tar -xz\n",
    "\n",
    "print_csv_preview(test_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def preprocess(text):\n",
    "  text = str(text)\n",
    "\n",
    "  # escape fasttext special sequences\n",
    "  text = text.replace(\"__label__\", \"\")\n",
    "\n",
    "  # lowercase\n",
    "  # text = text.lower()\n",
    "  \n",
    "  # remove html tags\n",
    "  # text = gensim.parsing.preprocessing.strip_tags(text)\n",
    "  \n",
    "  # remove punctuation\n",
    "  # text = gensim.parsing.preprocessing.strip_punctuation(text)\n",
    "  \n",
    "  # remove numerics\n",
    "  # text = gensim.parsing.preprocessing.strip_numeric(text)\n",
    "  \n",
    "  # remove consecutive whitespace characters and convert tabs to spaces\n",
    "  text = gensim.parsing.preprocessing.strip_multiple_whitespaces(text)\n",
    "  \n",
    "  # text = gensim.parsing.preprocessing.strip_short(text, minsize=3)\n",
    "  \n",
    "  # text = gensim.parsing.preprocessing.remove_stopwords(text)\n",
    "  \n",
    "  # text = gensim.parsing.preprocessing.stem_text(text)\n",
    "  \n",
    "  return text\n",
    "\n",
    "def preprocess_row(row):\n",
    "  doc = preprocess(row[\"title\"])\n",
    "  doc += \" \" + preprocess(row[\"body\"])\n",
    "\n",
    "  return doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Transform to fastText format:  60%|██████    | 766081/1275881 [00:59<00:28, 17695.60it/s]"
     ]
    }
   ],
   "source": [
    "\n",
    "def transform_to_fasttext_format(i_path, o_path):\n",
    "\twith open(i_path, \"r\", newline='', encoding=\"utf-8\") as i_f, open(o_path, \"w\", encoding='utf-8') as o_f:\n",
    "\t\treader = csv.DictReader(i_f)\n",
    "\t\ttotal = count_csv_rows(i_path)\n",
    "\t\tfor row in tqdm(reader, desc=\"Transform to fastText format\", total=total):\n",
    "\t\t\to_f.write(f\"__label__{row['labels']} {preprocess_row(row)}\\n\")\n",
    "\n",
    "transform_to_fasttext_format(train_file, \"issues.train\")\n",
    "\n",
    "!wc -l \"issues.train\"\n",
    "!head -n 2 \"issues.train\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Read 170M words\n",
      "Number of words:  11143356\n",
      "Number of labels: 4\n",
      "Progress: 100.0% words/sec/thread: 1689349 lr:  0.000000 avg.loss:  0.462312 ETA:   0h 0m 0s100.0% words/sec/thread: 1689354 lr: -0.000001 avg.loss:  0.462312 ETA:   0h 0m 0s\n"
     ]
    }
   ],
   "source": [
    "# https://fasttext.cc/docs/en/python-module.html#train_supervised-parameters\n",
    "\n",
    "model = fasttext.train_supervised(\"issues.train\")\n",
    "model.save_model(f\"issues.bin\")\n",
    "\n",
    "# model.quantize()\n",
    "# model.save_model(f\"issues.ftz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39fa7aeb7d584d8f8e1930e5355e70c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Benchmarking Inference Performance: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=*= bug =*=\n",
      "precision:\t0.8771\n",
      "recall:\t\t0.9173\n",
      "F1 score:\t0.8967\n",
      "\n",
      "=*= feature =*=\n",
      "precision:\t0.8415\n",
      "recall:\t\t0.8621\n",
      "F1 score:\t0.8517\n",
      "\n",
      "=*= question =*=\n",
      "precision:\t0.6702\n",
      "recall:\t\t0.4555\n",
      "F1 score:\t0.5424\n",
      "\n",
      "=*= documentation =*=\n",
      "precision:\t0.7363\n",
      "recall:\t\t0.5011\n",
      "F1 score:\t0.5964\n",
      "\n",
      "=*= global =*=\n",
      "precision:\t0.8510\n",
      "recall:\t\t0.8510\n",
      "F1 score:\t0.8510\n"
     ]
    }
   ],
   "source": [
    "# confusion matrix\n",
    "y_true = []\n",
    "y_pred = []\n",
    "\n",
    "with open(test_file, newline='', encoding='utf-8') as f:\n",
    "  reader = csv.DictReader(f)\n",
    "  for row in tqdm(reader, desc=\"Benchmarking Inference Performance\"):\n",
    "    pred = model.predict(preprocess_row(row))[0][0][9:]\n",
    "    y_true.append(row[\"labels\"])\n",
    "    y_pred.append(pred)\n",
    "\n",
    "for label in [\"bug\", \"feature\", \"question\", \"documentation\"]:\n",
    "  P_c = sklearn.metrics.precision_score(y_true, y_pred, average=None, labels=[label])[0]\n",
    "  R_c = sklearn.metrics.recall_score(y_true, y_pred, average=None, labels=[label])[0]\n",
    "  F1_c = sklearn.metrics.f1_score(y_true, y_pred, average=None, labels=[label])[0]\n",
    "  print(f\"=*= {label} =*=\")\n",
    "  print(f\"precision:\\t{P_c:.4f}\")\n",
    "  print(f\"recall:\\t\\t{R_c:.4f}\")\n",
    "  print(f\"F1 score:\\t{F1_c:.4f}\")\n",
    "  print()\n",
    "\n",
    "\n",
    "P = sklearn.metrics.precision_score(y_true, y_pred, average='micro')\n",
    "R = sklearn.metrics.recall_score(y_true, y_pred, average='micro')\n",
    "F1 = sklearn.metrics.f1_score(y_true, y_pred, average='micro')\n",
    "\n",
    "print(\"=*= micro averages =*=\")\n",
    "print(f\"precision:\\t{P:.4f}\")\n",
    "print(f\"recall:\\t\\t{R:.4f}\")\n",
    "print(f\"F1 score:\\t{F1:.4f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}