evaluate_all_models_performance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1bb0aca5-dcbf-47c1-ba8e-faa47cc7be6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Import training data\n",
    "# split into train and test sets\n",
    "import pandas as pd\n",
    "import csv\n",
    "from train_inlegalbert_xgboost import load_text_data, split_data\n",
    "train_texts, test_texts, train_labels, test_labels = load_text_data('complete_training_data_7200_cases.csv')\n",
    "model_path = Path('inlegal_bert_xgboost_classifier.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d32ec30e-4db9-43d4-97b0-a8ccc396346a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data written to test_set_1451.csv successfully.\n"
     ]
    }
   ],
   "source": [
    "# 2. Write test data to file\n",
    "# Path to the output CSV file\n",
    "csv_file_path = 'test_set_1451.csv'\n",
    "\n",
    "# Ensure both lists have the same length\n",
    "if len(test_texts) != len(test_labels):\n",
    "    raise ValueError(\"Both lists must have the same length\")\n",
    "\n",
    "# Writing to the CSV file\n",
    "with open(csv_file_path, mode='w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    # Write the header\n",
    "    writer.writerow([\"sentence\", \"ground_truth\"])\n",
    "    # Write the rows\n",
    "    for item1, item2 in zip(test_texts, test_labels):\n",
    "        writer.writerow([item1, item2])\n",
    "\n",
    "print(f\"Data written to {csv_file_path} successfully.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "806a9f89-13a8-4d3a-be9f-5146900d7328",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Execute dependency parser classification on test set (offline using rule-based-classification.py script)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "136698ff-2757-48af-8fbd-527646d1b255",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. Load classified results\n",
    "classified_df = pd.read_csv('rule_classified_1451.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c1875a34-c0f5-4110-a62b-a76181fc86bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   recall  precision    f1\n",
      "0    0.89       0.76  0.82\n",
      "1    0.70       0.86  0.77\n",
      "                    score\n",
      "accuracy             0.80\n",
      "krippendorff alpha   0.59\n",
      "        Predicted:     \n",
      "                 0    1\n",
      "True: 0        663   80\n",
      "      1        211  497\n"
     ]
    }
   ],
   "source": [
    "# 5. Print performance metrics\n",
    "from performance_metrics import print_performance_metrics\n",
    "print_performance_metrics(classified_df['ground_truth'].tolist(), classified_df['regulatory_according_to_rule'].tolist(), None, [0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ba571e59-a31d-4f11-9eec-8390496e17a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. Add ML model classification results as well\n",
    "from train_inlegalbert_xgboost import class_names\n",
    "import json\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from classify_text_with_inlegal_bert_xgboost import classify_texts\n",
    "from dianna.utils.tokenizers import SpacyTokenizer\n",
    "\n",
    "class StatementClassifier:\n",
    "    def __init__(self):\n",
    "        self.tokenizer = SpacyTokenizer(name='en_core_web_sm')\n",
    "\n",
    "    def __call__(self, sentences):\n",
    "        # ensure the input has a batch axis\n",
    "        if isinstance(sentences, str):\n",
    "            sentences = [sentences]\n",
    "\n",
    "        probs = classify_texts(sentences, model_path, return_proba=True)\n",
    "\n",
    "        return np.transpose([(probs[:, 0]), (1 - probs[:, 0])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "aec689ca-26cd-43ed-aad6-80fab1c8a1e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_runner = StatementClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a5889c88-be1f-4035-8488-bf6b40816d5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kodymoodley/Desktop/regulatory-statement-classification/regenv/lib/python3.9/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Some weights of the model checkpoint at law-ai/InLegalBERT were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Creating features: 100%|███████████████████████████████████████████████| 1451/1451 [01:00<00:00, 23.89it/s]\n"
     ]
    }
   ],
   "source": [
    "prediction = model_runner(classified_df['sent'].tolist())\n",
    "ml_model_classification_results = [m for m in np.argmax(prediction, axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c97d589b-3e51-40ca-8049-85c94f1189c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "classified_df['ml_model_result'] = ml_model_classification_results\n",
    "classified_df.to_csv('all_models_and_algorithms_combined_results_test_set_1451.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "regenv",
   "language": "python",
   "name": "regenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}