train_stage_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "# Automatically reload modules\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(73233, 23)\n",
      "(54069, 23) (19164, 23)\n",
      "['anon_id' 'exam_id' 'Age_exam' 'Sex' 'Weight' 'Length' 'diagnosis'\n",
      " 'muscle' 'side' 'z_score' 'h_score' 'image_file' 'has_markers' 'li_x'\n",
      " 'li_y' 're_x' 're_y' 'id_x' 'id_y' 'bmi' 'prediction'\n",
      " 'rounded_prediction' 'label']\n",
      "h_score\n",
      "1.0    49772\n",
      "2.0    16420\n",
      "3.0     6433\n",
      "4.0      608\n",
      "Name: count, dtype: int64\n",
      "patient count: 1475\n",
      "exam count: 1774\n",
      "diagnosis\n",
      "5     32956\n",
      "0     17024\n",
      "1      7112\n",
      "99     6707\n",
      "2      5855\n",
      "3      3277\n",
      "4       281\n",
      "6        21\n",
      "Name: count, dtype: int64\n",
      "(66526, 23)\n",
      "diagnosis\n",
      "Diseased    780\n",
      "Healthy     530\n",
      "Name: count, dtype: int64\n",
      "Total count: 1310\n",
      "Healthy: 530 (40.46%)\n",
      "Diseased: 780 (59.54%)\n"
     ]
    }
   ],
   "source": [
    "#data_dir = \"/Users/peter/repos/qu/results/2024-06-13_17-02-36_masked-image-text/output\"\n",
    "#session_label = \"2024-11-21_15-42-26_with-cm-logging6\"\n",
    "session_label = \"2024-12-05_16-34-39_output-test-predictions\"\n",
    "output_epoch = \"epoch_18\"\n",
    "data_dir = f\"/Users/peter/repos/qu/results/{session_label}/output\"\n",
    "if not output_epoch == \"\":\n",
    "    data_dir = os.path.join(data_dir, output_epoch)\n",
    "\n",
    "# Read and concatenate the data sets (just use all of them for now)\n",
    "df1 = pd.read_csv(os.path.join(data_dir, \"df_train_predictions.csv\"))\n",
    "df2 = pd.read_csv(os.path.join(data_dir, \"df_validation_predictions.csv\"))\n",
    "df3 = pd.read_csv(os.path.join(data_dir, \"df_test_predictions.csv\"))\n",
    "df_combined = pd.concat([df1, df2])\n",
    "print(df_combined.shape)\n",
    "print(df1.shape, df2.shape)\n",
    "print(df_combined.columns.values)\n",
    "\n",
    "print(df_combined['h_score'].value_counts())\n",
    "\n",
    "print('patient count:', df_combined['anon_id'].nunique())\n",
    "print('exam count:', df_combined['exam_id'].nunique())\n",
    "\n",
    "# Remove diagnosis 99 (ignored in 2nd stage model)\n",
    "print(df_combined['diagnosis'].value_counts())\n",
    "df_combined = df_combined[df_combined['diagnosis'] != 99]\n",
    "\n",
    "print(df_combined.shape)\n",
    "\n",
    "# Drop duplicate entries based on 'patient_id'\n",
    "unique_patients = df_combined.drop_duplicates(subset='anon_id')\n",
    "total_count = unique_patients.shape[0]\n",
    "\n",
    "# Count the number of healthy (0) and diseased (non-0) patients\n",
    "diagnosis_counts = unique_patients['diagnosis'].apply(lambda x: 'Healthy' if x == 0 else 'Diseased').value_counts()\n",
    "print(diagnosis_counts)\n",
    "healthy_count = diagnosis_counts['Healthy']\n",
    "diseased_count = diagnosis_counts['Diseased']\n",
    "# percentages\n",
    "print(f\"Total count: {total_count}\")\n",
    "print(f\"Healthy: {healthy_count} ({100*healthy_count/total_count:.2f}%)\")\n",
    "print(f\"Diseased: {diseased_count} ({100*diseased_count/total_count:.2f}%)\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "muscle\n",
      "Biceps        8207\n",
      "TA            7774\n",
      "GM            7716\n",
      "VL            6244\n",
      "Deltoid       5504\n",
      "FCR           5257\n",
      "RF            4960\n",
      "Rectusab      3771\n",
      "Trap          3338\n",
      "ID1           3006\n",
      "Masseter      2957\n",
      "Digastric     2919\n",
      "PerT          1794\n",
      "SCM           1566\n",
      "Geniohyoid    1463\n",
      "Extensors       50\n",
      "Name: count, dtype: int64\n",
      "{'Biceps': 0, 'RF': 1, 'TA': 2, 'GM': 3, 'VL': 4, 'Deltoid': 5}\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "input_column = 'muscle'\n",
    "input_value_column = 'prediction'\n",
    "print(df_combined[input_column].value_counts())\n",
    "#input_values = df[input_column].unique() # all muscles\n",
    "#input_values = input_values[input_values != 'Extensors'] # remove Extensors\n",
    "input_values = ['Biceps','RF','TA','GM','VL','Deltoid'] # six most common muscles\n",
    "value_to_index = {value: idx for idx, value in enumerate(input_values)}\n",
    "print(value_to_index)\n",
    "\n",
    "# Output\n",
    "use_diagnosis = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                            exam_id  muscle  prediction  h_score  diagnosis\n",
      "0  0037dc33a92f6d45871c3602a20ac322  Biceps    1.959257      1.0          0\n",
      "1  0037dc33a92f6d45871c3602a20ac322     FCR    1.763558      1.0          0\n",
      "2  0037dc33a92f6d45871c3602a20ac322      GM    1.711965      1.0          0\n",
      "3  0037dc33a92f6d45871c3602a20ac322      VL    2.210034      1.0          0\n",
      "4  0037e1e7de4d3ee36ac5d1c9b3ab8073  Biceps    1.923798      1.0          5\n",
      "                            exam_id    muscle  prediction  h_score  diagnosis\n",
      "0  0084fbea6e3650abd91ce4d3ee4d4699    Biceps    1.708533      2.0          5\n",
      "1  0084fbea6e3650abd91ce4d3ee4d4699   Deltoid    1.517465      2.0          5\n",
      "2  0084fbea6e3650abd91ce4d3ee4d4699        GM    2.897208      3.0          5\n",
      "3  0084fbea6e3650abd91ce4d3ee4d4699        RF    1.747876      2.0          5\n",
      "4  0084fbea6e3650abd91ce4d3ee4d4699  Rectusab    2.517725      2.0          5\n",
      "                            exam_id   muscle  prediction  h_score  diagnosis\n",
      "0  0035bbf8a6ab0b0247b43ea33689aa46   Biceps    1.999800      2.0          5\n",
      "1  0035bbf8a6ab0b0247b43ea33689aa46  Deltoid    1.308979      1.0          5\n",
      "2  0035bbf8a6ab0b0247b43ea33689aa46      FCR    1.474484      1.0          5\n",
      "3  0035bbf8a6ab0b0247b43ea33689aa46       GM    1.985300      1.0          5\n",
      "4  0035bbf8a6ab0b0247b43ea33689aa46       RF    2.284842      2.0          5\n"
     ]
    }
   ],
   "source": [
    "# Aggregate the h_scores for each muscle\n",
    "df_agg_train = df1.groupby(['exam_id', input_column]).agg({'prediction': 'max', 'h_score': 'max', 'diagnosis': 'first'}).reset_index()\n",
    "df_agg_val = df2.groupby(['exam_id', input_column]).agg({'prediction': 'max', 'h_score': 'max', 'diagnosis': 'first'}).reset_index()\n",
    "df_agg_test = df3.groupby(['exam_id', input_column]).agg({'prediction': 'max', 'h_score': 'max', 'diagnosis': 'first'}).reset_index()\n",
    "# df_agg['h_score'] = np.random.normal(2, 1.0, df_agg.shape[0])\n",
    "# df_agg['h_score'] = 1.0\n",
    "print(df_agg_train.head(5))\n",
    "print(df_agg_val.head(5))\n",
    "print(df_agg_test.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1316, 6) (1316, 1)\n",
      "(458, 6) (458, 1)\n",
      "(412, 6) (412, 1)\n"
     ]
    }
   ],
   "source": [
    "def create_vectors(df):\n",
    "    input_vectors = {id: np.full(len(input_values), np.NaN) for id in df['exam_id'].unique()}\n",
    "    label_vectors = {id: False for id in df['exam_id'].unique()}\n",
    "\n",
    "    # Populate the vectors\n",
    "    for _, row in df.iterrows():\n",
    "        exam_id = row['exam_id']\n",
    "        muscle = row[input_column]\n",
    "        if muscle in input_values:\n",
    "            value_index = value_to_index[muscle]\n",
    "            input_vectors[exam_id][value_index] = row[input_value_column]\n",
    "            #diagnosis_index = diagnosis_to_index[row['diagnosis']]\n",
    "            if use_diagnosis:\n",
    "                label_vectors[exam_id] = row['diagnosis']\n",
    "            else:\n",
    "                label_vectors[exam_id] = row['diagnosis'] != 0\n",
    "\n",
    "    # Convert the dictionary to a DataFrame for easy handling/viewing\n",
    "    input_df = pd.DataFrame.from_dict(input_vectors, orient='index', columns=input_values)\n",
    "    label_df = pd.DataFrame.from_dict(label_vectors, orient='index', columns=['diagnosis'])\n",
    "    return (input_df, label_df)\n",
    "\n",
    "input_train_df, label_train_df = create_vectors(df_agg_train)\n",
    "input_val_df, label_val_df = create_vectors(df_agg_val)\n",
    "input_test_df, label_test_df = create_vectors(df_agg_test)\n",
    "\n",
    "# Handling missing values in 'input_df' (simple imputation with mean of each column)\n",
    "# input_df = input_df.apply(lambda x: x.fillna(x.mean()), axis=1)\n",
    "# input_df = input_df.fillna(0)\n",
    "#input_df_filled = pd.DataFrame(0, index=np.arange(input_df.shape[0]), columns=input_df.columns)\n",
    "#print(input_df_filled.head(10))\n",
    "\n",
    "print(input_train_df.shape, label_train_df.shape)\n",
    "print(input_val_df.shape, label_val_df.shape)\n",
    "print(input_test_df.shape, label_test_df.shape)\n",
    "# display(input_df.head(20))\n",
    "# display(label_df.head(20))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Biceps', 'RF', 'TA', 'GM', 'VL', 'Deltoid'] ['False', 'True']\n"
     ]
    }
   ],
   "source": [
    "label_values = label_train_df['diagnosis'].unique()\n",
    "label_values = [str(x) for x in label_values]\n",
    "print(input_values, label_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1316, 6, 2)\n",
      "(458, 6, 2)\n",
      "(412, 6, 2)\n"
     ]
    }
   ],
   "source": [
    "def df_to_masked_array(df):\n",
    "    # Convert the dataframe to a numpy array\n",
    "    array = df.to_numpy()\n",
    "\n",
    "    # Create a mask array, 1.0 where the original array is not NaN, and 0.0 where it is NaN\n",
    "    mask = np.where(np.isnan(array), 0.0, 1.0)\n",
    "\n",
    "    # Stack the original array and the mask array to form a new array with 2 channels\n",
    "    two_channel_array = np.stack((array, mask), axis=-1)\n",
    "\n",
    "    # Replace NaN values in the original data with 0.0 (or another value of your choice)\n",
    "    two_channel_array[np.isnan(two_channel_array[..., 0]), 0] = 0.0\n",
    "\n",
    "    return two_channel_array\n",
    "\n",
    "X_train_masked = df_to_masked_array(input_train_df)\n",
    "print(X_train_masked.shape)\n",
    "X_val_masked = df_to_masked_array(input_val_df)\n",
    "print(X_val_masked.shape)\n",
    "X_test_masked = df_to_masked_array(input_test_df)\n",
    "print(X_test_masked.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def boolean_df_to_int_array(df):\n",
    "    return df.astype(int).to_numpy()\n",
    "\n",
    "if use_diagnosis:\n",
    "    y_train_int = label_train_df.astype(int).to_numpy()\n",
    "    y_val_int = label_val_df.astype(int).to_numpy()\n",
    "    y_test_int = label_test_df.astype(int).to_numpy()\n",
    "else:\n",
    "    y_train_int = boolean_df_to_int_array(label_train_df)\n",
    "    y_val_int = boolean_df_to_int_array(label_val_df)\n",
    "    y_test_int = boolean_df_to_int_array(label_test_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "X_train_tensor = torch.tensor(X_train_masked, dtype=torch.float).permute(0, 2, 1)  # Swap the last two dimensions\n",
    "y_train_tensor = torch.tensor(y_train_int, dtype=torch.long)\n",
    "X_val_tensor = torch.tensor(X_val_masked, dtype=torch.float).permute(0, 2, 1)  # Swap the last two dimensions\n",
    "y_val_tensor = torch.tensor(y_val_int, dtype=torch.long)\n",
    "X_test_tensor = torch.tensor(X_test_masked, dtype=torch.float).permute(0, 2, 1)  # Swap the last two dimensions\n",
    "y_test_tensor = torch.tensor(y_test_int, dtype=torch.long)\n",
    "\n",
    "# Create DataLoader for batching\n",
    "train_data = torch.utils.data.TensorDataset(X_train_tensor, y_train_tensor)\n",
    "train_loader = DataLoader(train_data, batch_size=32, shuffle=True)\n",
    "val_data = torch.utils.data.TensorDataset(X_val_tensor, y_val_tensor)\n",
    "val_loader = DataLoader(val_data, batch_size=32, shuffle=False)\n",
    "test_data = torch.utils.data.TensorDataset(X_test_tensor, y_test_tensor)\n",
    "test_loader = DataLoader(test_data, batch_size=32, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv1d-1                [-1, 64, 6]             192\n",
      "            Linear-2                   [-1, 64]          24,640\n",
      "            Linear-3                    [-1, 1]              65\n",
      "================================================================\n",
      "Total params: 24,897\n",
      "Trainable params: 24,897\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.00\n",
      "Params size (MB): 0.09\n",
      "Estimated Total Size (MB): 0.10\n",
      "----------------------------------------------------------------\n",
      "Initial loss: 0.4368986040353775\n",
      "Epoch [1/20], Loss: 0.3596, Val Loss: 0.3675\n",
      "Epoch [2/20], Loss: 0.3532, Val Loss: 0.3575\n",
      "Epoch [3/20], Loss: 0.3622, Val Loss: 0.3649\n",
      "Epoch [4/20], Loss: 0.3442, Val Loss: 0.3535\n",
      "Epoch [5/20], Loss: 0.3718, Val Loss: 0.3685\n",
      "Epoch [6/20], Loss: 0.3612, Val Loss: 0.3644\n",
      "Epoch [7/20], Loss: 0.3817, Val Loss: 0.3788\n",
      "Epoch [8/20], Loss: 0.3756, Val Loss: 0.3701\n",
      "Epoch [9/20], Loss: 0.3686, Val Loss: 0.3623\n",
      "Epoch [10/20], Loss: 0.3721, Val Loss: 0.3651\n",
      "Epoch [11/20], Loss: 0.3814, Val Loss: 0.3670\n",
      "Epoch [12/20], Loss: 0.3777, Val Loss: 0.3728\n",
      "Epoch [13/20], Loss: 0.3662, Val Loss: 0.3468\n",
      "Epoch [14/20], Loss: 0.3777, Val Loss: 0.3662\n",
      "Epoch [15/20], Loss: 0.3728, Val Loss: 0.3675\n",
      "Epoch [16/20], Loss: 0.4009, Val Loss: 0.3634\n",
      "Epoch [17/20], Loss: 0.4059, Val Loss: 0.3756\n",
      "Epoch [18/20], Loss: 0.4014, Val Loss: 0.3728\n",
      "Epoch [19/20], Loss: 0.4066, Val Loss: 0.3884\n",
      "Epoch [20/20], Loss: 0.4211, Val Loss: 0.3856\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from qumia_model2 import MaskedClassifier\n",
    "from torchsummary import summary\n",
    "\n",
    "# Define the model, loss function, and optimizer\n",
    "vector_length = len(input_values)\n",
    "num_classes = len(label_values) if use_diagnosis else 1\n",
    "model = MaskedClassifier(vector_length, num_classes)\n",
    "criterion = nn.CrossEntropyLoss() if use_diagnosis else nn.BCEWithLogitsLoss(pos_weight=torch.tensor([0.45]))\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.005)\n",
    "\n",
    "# print model summary\n",
    "summary(model, (2, vector_length))\n",
    "\n",
    "# Calculate losses\n",
    "total_loss = 0.0\n",
    "for inputs, labels in train_loader:\n",
    "    outputs = model(inputs)\n",
    "    target = labels.squeeze() if use_diagnosis else labels.float()\n",
    "    loss = criterion(outputs, target)\n",
    "    total_loss += loss.item()\n",
    "print(f'Initial loss: {total_loss/len(train_loader)}')\n",
    "\n",
    "# Training the model\n",
    "num_epochs = 20  # Set the number of epochs\n",
    "for epoch in range(num_epochs):\n",
    "    for inputs, labels in train_loader:\n",
    "        # Forward pass\n",
    "        outputs = model(inputs)\n",
    "        target = labels.squeeze() if use_diagnosis else labels.float()\n",
    "        loss = criterion(outputs, target)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    \n",
    "    # Calculate loss on validation set\n",
    "    with torch.no_grad():\n",
    "        val_loss = 0.0\n",
    "        for inputs, labels in val_loader:\n",
    "            outputs = model(inputs)\n",
    "            target = labels.squeeze() if use_diagnosis else labels.float()\n",
    "            loss = criterion(outputs, target)\n",
    "            val_loss += loss.item()\n",
    "        val_loss /= len(val_loader)\n",
    "    \n",
    "    # Save the checkpoint\n",
    "    torch.save(model.state_dict(), f'checkpoint_{epoch+1}.pth')\n",
    "\n",
    "    print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}, Val Loss: {val_loss:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "# model.eval()  # Set the model to evaluation mode\n",
    "# with torch.no_grad():\n",
    "#     test_preds = model(X_test_tensor)\n",
    "#     _, predicted_classes = torch.max(test_preds, 1)\n",
    "#     matches = (predicted_classes == y_test_tensor.squeeze())\n",
    "#     accuracy = matches.sum().float().item() / y_test_tensor.size(0)\n",
    "#     print(f'Accuracy: {accuracy:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "# Load the model at a specific checkpoint\n",
    "model.load_state_dict(torch.load('checkpoint_4.pth'))\n",
    "\n",
    "# Set the model to evaluation mode\n",
    "model.eval()\n",
    "\n",
    "# Store all the predictions and true labels\n",
    "test_predictions = []\n",
    "test_labels = []\n",
    "correct_predictions = 0\n",
    "total_predictions = 0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for inputs, labels in test_loader:\n",
    "        outputs = model(inputs)\n",
    "\n",
    "        if use_diagnosis:\n",
    "            # Get the predicted classes by finding the max logit value\n",
    "            _, predicted_classes = torch.max(outputs, 1)\n",
    "            \n",
    "            # Collect the predictions and true labels\n",
    "            test_predictions.extend(predicted_classes.tolist())\n",
    "            test_labels.extend(labels.tolist())\n",
    "\n",
    "            correct_predictions += (predicted_classes == labels.squeeze()).sum().item()\n",
    "            total_predictions += labels.size(0)\n",
    "\n",
    "        else:\n",
    "\n",
    "            # Apply sigmoid to the outputs to get the probabilities\n",
    "            probs = torch.sigmoid(outputs.squeeze())  # Squeeze to remove unnecessary dimensions\n",
    "            # Convert probabilities to binary predictions using a threshold of 0.5\n",
    "            \n",
    "            #predictions = (probs >= 0.5).long()  # Convert to long to match labels type\n",
    "            \n",
    "            # Collect the predictions and true labels (for f1 score)\n",
    "            test_predictions.extend(probs.tolist())\n",
    "            test_labels.extend(labels.tolist())\n",
    "\n",
    "            # Compare predictions to true labels (for accuracy)\n",
    "            #correct_predictions += (predictions == labels.squeeze()).sum().item()\n",
    "            #total_predictions += labels.size(0)\n",
    "\n",
    "if use_diagnosis:\n",
    "    # Calculate the F1 score (choose 'macro', 'micro', or 'weighted' as per your requirement)\n",
    "    f1_macro = f1_score(test_labels, test_predictions, average='macro')\n",
    "    f1_micro = f1_score(test_labels, test_predictions, average='micro')\n",
    "    f1_weighted = f1_score(test_labels, test_predictions, average='weighted')\n",
    "\n",
    "    print(f'F1 Score (Macro): {f1_macro:.4f}')\n",
    "    print(f'F1 Score (Micro): {f1_micro:.4f}')\n",
    "    print(f'F1 Score (Weighted): {f1_weighted:.4f}')\n",
    "    \n",
    "    from sklearn.metrics import classification_report\n",
    "    # all_classes = np.array(label_values)\n",
    "    # unique_labels = np.unique(np.concatenate((all_classes, all_predictions)))\n",
    "    # print(unique_labels)\n",
    "    report = classification_report(test_labels, test_predictions)\n",
    "    print(report)\n",
    "    \n",
    "else:\n",
    "    pass\n",
    "    # Calculate the F1 score\n",
    "    #f1 = f1_score(all_labels, all_predictions, average='binary')  # 'binary' for binary classification tasks\n",
    "    #print(f'F1 Score: {f1:.4f}')\n",
    "\n",
    "#print(f'Accuracy: {correct_predictions / total_predictions:.4f}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# Assuming all_predictions and all_labels are your predicted and true labels, respectively\n",
    "\n",
    "# Generate the confusion matrix\n",
    "# cm = confusion_matrix(all_labels, all_predictions)\n",
    "\n",
    "# # Print the confusion matrix\n",
    "# print(\"Confusion Matrix:\")\n",
    "# col_labels = ['', 'Pred 0', 'Pred 1']\n",
    "# pred_labels = ['True 0', 'True 1']\n",
    "# for i, label in enumerate(col_labels):\n",
    "#     print(f\"{label:>8}\", end=\" \")\n",
    "# print()\n",
    "# for i, row in enumerate(cm):\n",
    "#     print(f\"{pred_labels[i]:>8}\", end=\" \")\n",
    "#     for val in row:\n",
    "#         print(f\"{val:>8}\", end=\" \")\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAHHCAYAAABTMjf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABX80lEQVR4nO3deVhUZf8G8HsYmAWGVXZEUXFfUFH5oRlqJGpZvpVamVtZllomlWmlaKZomkuupW9qpWlamuWWouZarvhqghsoiKwqDOsMzDy/P5DJCVD2Aeb+XNdcMc8855zv4aRze85zniMRQggQERERmSELUxdAREREZCoMQkRERGS2GISIiIjIbDEIERERkdliECIiIiKzxSBEREREZotBiIiIiMwWgxARERGZLQYhIiIiMlsMQkS13KhRo+Dj41OuZQ4dOgSJRIJDhw5VS011Xa9evdCrVy/D+xs3bkAikWDdunUmq4mITINBiOhf1q1bB4lEYngpFAq0aNECEyZMQHJysqnLq/WKQkXRy8LCAk5OTujfvz9OnDhh6vKqlU6ng6enJyQSCXbv3l1in1GjRkGlUpW6DpVKhVGjRhVrT05Oxvvvv49WrVrB2toaNjY28Pf3x2effYb09PQqqf/48eN47LHHYG1tDXd3d7zzzjvIysoq07IZGRmYPHkymjdvDqVSicaNG+O1115DXFycUT8fHx+j/z8efDVv3rxK9oOoPCxNXQBRbfXpp5+iSZMmyMvLw9GjR7Fy5Urs2rULFy9ehLW1dY3VsXr1auj1+nIt8/jjjyM3Nxcymayaqnq0l156CQMGDIBOp8OVK1ewYsUK9O7dG6dOnUL79u1NVld1OnDgABITE+Hj44MNGzagf//+VbLeU6dOYcCAAcjKysIrr7wCf39/AMDp06cxd+5cHD58GL///nulthEZGYknnngCrVu3xsKFC3Hr1i0sWLAAV69eLTXUFdHr9XjyySdx6dIljBs3Di1atMC1a9ewYsUK7N27F1FRUbC1tQUALF68uFi4unnzJj755BP07du3UvtAVCGCiIysXbtWABCnTp0yag8NDRUAxMaNG0tdNisrq7rLq/ViY2MFADF//nyj9t27dwsA4q233jJRZf8ICgoSQUFBhvdFNa9du7ZS6x0xYoTo3LmzWLJkibCxsSnx/4eRI0cKGxubUtdhY2MjRo4caXh/79494eXlJdzc3ERUVFSx/klJSWLWrFmVqlsIIfr37y88PDxERkaGoW316tUCgNi7d+9Dlz127JgAIJYtW2bU/s033wgA4ueff37o8rNmzRIAxLFjxyq+A0QVxEtjRGXUp08fAEBsbCyAfy5xXL9+HQMGDICtrS2GDRsGoPBfyIsXL0bbtm2hUCjg5uaGsWPH4t69e8XWu3v3bgQFBcHW1hZ2dnbo2rUrNm7caPi8pDFCmzZtgr+/v2GZ9u3bY8mSJYbPSxsjtGXLFvj7+0OpVMLZ2RmvvPIKEhISjPoU7VdCQgIGDRoElUoFFxcXvP/++9DpdBX+/fXs2RMAcP36daP29PR0vPvuu/D29oZcLoevry/mzZtX7CyYXq/HkiVL0L59eygUCri4uKBfv344ffq0oc/atWvRp08fuLq6Qi6Xo02bNli5cmWFay6P3NxcbNu2DS+++CKGDBmC3Nxc/PLLL5Ve71dffYWEhAQsXLgQrVq1Kva5m5sbPvnkk0ptQ61WY9++fXjllVdgZ2dnaB8xYgRUKhV+/PHHRy5fVMuDPDw8AABKpfKhy2/cuBFNmjRB9+7dK1I+UaXw0hhRGRV9gTdo0MDQVlBQgJCQEDz22GNYsGCB4ZLZ2LFjsW7dOowePRrvvPMOYmNjsWzZMpw7dw7Hjh2DlZUVgMLxSK+++iratm2LqVOnwsHBAefOncOePXvw8ssvl1jHvn378NJLL+GJJ57AvHnzAABRUVE4duwYJk6cWGr9RfV07doV4eHhSE5OxpIlS3Ds2DGcO3cODg4Ohr46nQ4hISEICAjAggULsH//fnzxxRdo1qwZ3nrrrQr9/m7cuAEAcHR0NLTl5OQgKCgICQkJGDt2LBo1aoTjx49j6tSpSExMxOLFiw19X3vtNaxbtw79+/fHmDFjUFBQgCNHjuDPP/9Ely5dAAArV65E27Zt8cwzz8DS0hK//vorxo0bB71ej/Hjx1eo7rLasWMHsrKy8OKLL8Ld3R29evXChg0bSj2O5VmvUqnECy+8UKb+WVlZyMvLe2Q/Kysr2NvbAwAuXLiAgoICw++xiEwmQ8eOHXHu3LmHrqtLly6wsbHBtGnT4OTkhJYtW+LatWuYPHkyunbtiuDg4FKXPXfuHKKiovDxxx+XYe+IqoGpT0kR1TZFl8b2798vUlNTRXx8vNi0aZNo0KCBUCqV4tatW0KIwkscAMSUKVOMlj9y5IgAIDZs2GDUvmfPHqP29PR0YWtrKwICAkRubq5RX71eb/h55MiRonHjxob3EydOFHZ2dqKgoKDUfTh48KAAIA4ePCiEEEKr1QpXV1fRrl07o2399ttvAoCYPn260fYAiE8//dRonZ06dRL+/v6lbrNI0WWmmTNnitTUVJGUlCSOHDkiunbtKgCILVu2GPrOmjVL2NjYiCtXrhitY8qUKUIqlYq4uDghhBAHDhwQAMQ777xTbHsP/q5ycnKKfR4SEiKaNm1q1FYdl8aefvpp0aNHD8P7r7/+WlhaWoqUlBSjfuW9NObo6Cj8/PzKXEfR8XvU68H937JliwAgDh8+XGx9gwcPFu7u7o/c7m+//SY8PDyMthESEiIyMzMfutx7770nAIhLly6VeR+JqhIvjRGVIjg4GC4uLvD29saLL74IlUqFbdu2wcvLy6jfv8+QbNmyBfb29njyySeRlpZmePn7+0OlUuHgwYMACs/sZGZmYsqUKVAoFEbrkEgkpdbl4OCA7Oxs7Nu3r8z7cvr0aaSkpGDcuHFG23rqqafQqlUr7Ny5s9gyb775ptH7nj17IiYmpszbDAsLg4uLC9zd3dGzZ09ERUXhiy++MDqzsWXLFvTs2ROOjo5Gv6vg4GDodDocPnwYAPDTTz9BIpEgLCys2HYe/F09eAkmIyMDaWlpCAoKQkxMDDIyMspce3nduXMHe/fuxUsvvWRoe/755yGRSB55WelR1Gq1YaBxWUyePBn79u175OuLL74wLJObmwsAkMvlxdanUCgMnz+Mi4sLOnXqhNmzZ2P79u2YMWMGjhw5gtGjR5e6jF6vx6ZNm9CpUye0bt26zPtIVJV4aYyoFMuXL0eLFi1gaWkJNzc3tGzZEhYWxv92sLS0RMOGDY3arl69ioyMDLi6upa43pSUFAD/XGpr165dueoaN24cfvzxR/Tv3x9eXl7o27cvhgwZgn79+pW6zM2bNwEALVu2LPZZq1atcPToUaO2ojE4D3J0dDQa45Sammo0ZkilUhndFv7GG29g8ODByMvLw4EDB/Dll18WG2N09epV/O9//yu2rSIP/q48PT3h5ORU6j4CwLFjxxAWFoYTJ04gJyfH6LOMjAzDpaCqtnnzZuTn56NTp064du2aoT0gIAAbNmwo92W5B8OdnZ0dMjMzy7xsmzZt0KZNm3JtryhAajSaYp/l5eU9coxPTEwMevfujW+//RbPP/88AODZZ5+Fj48PRo0ahd27d5d4B90ff/yBhIQETJo0qVz1ElUlBiGiUnTr1q3YmIl/k8vlxcKRXq+Hq6srNmzYUOIypX3pl5WrqysiIyOxd+9e7N69G7t378batWsxYsQIrF+/vlLrLiKVSh/Zp2vXroaABRSeAZoxY4bhffPmzQ1jQ55++mlIpVJMmTIFvXv3Nvxei267njx5conbaNGiRZlrvn79Op544gm0atUKCxcuhLe3N2QyGXbt2oVFixaVewqC8ig61j169Cjx85iYGDRt2hRAYcjUaDQQQhQ78yeEQF5entFZu1atWiEyMhJarbZM0yFkZGSU6QyOTCYzBMuiQc2JiYnF+iUmJsLT0/Oh61q3bh3y8vLw9NNPG7U/88wzAAoDaklBaMOGDbCwsDA6k0ZU0xiEiKpYs2bNsH//fvTo0eOh/5Ju1qwZAODixYvw9fUt1zZkMhkGDhyIgQMHQq/XY9y4cfjqq68wbdq0EtfVuHFjAMDly5cNd78VuXz5suHz8tiwYYPRF27RF31pPv74Y6xevRqffPIJ9uzZA6Dwd5CVlfXQwbRF/fbu3Yu7d++Welbo119/hUajwY4dO9CoUSNDe9GlyOoSGxuL48ePY8KECQgKCjL6TK/XY/jw4di4caPhzq7GjRujoKAA169fL3asrl27Bp1OZ3Q8Bg4ciBMnTuCnn34qU2CYOHFimQJxUFCQ4a7Cdu3awdLSEqdPn8aQIUMMfbRaLSIjI43aSpKcnAwhRLEzfvn5+QAKbyr4N41Gg59++gm9evV6ZNAiqk4cI0RUxYYMGQKdTodZs2YV+6ygoMAwC3Dfvn1ha2uL8PDwYnf5CCFKXf+dO3eM3ltYWKBDhw4ASr60ARTe1ePq6opVq1YZ9dm9ezeioqLw1FNPlWnfHtSjRw8EBwcbXo8KQg4ODhg7diz27t2LyMhIAIW/qxMnTmDv3r3F+qenpxu+QJ9//nkIITBz5sxi/Yp+V0VnsR783WVkZGDt2rXl3rfyKDobNHnyZLzwwgtGryFDhiAoKMjo7GDRmZFly5YVW9fy5cuN+gCFY7U8PDzw3nvv4cqVK8WWSUlJwWeffWZ4X5ExQvb29ggODsb3339vdBnuu+++Q1ZWFgYPHmxoy8nJQXR0NNLS0gxtLVq0gBCi2HioH374AQDQqVOnYnXv2rUL6enphikniEzGdOO0iWqn0iZU/LeH3f0zduxYAUD0799fLFq0SCxbtkxMnDhReHp6Gt01tWbNGgFAtGvXTsyZM0esXLlSvPnmm2LEiBFG23nwrrFBgwaJxx9/XMyYMUOsWbNGTJs2TTg4OIiOHTsKnU4nhCh+19iD+xUQECAWL14spk6dKqytrYWPj4+4d+/eI/crLCxMlOWvjNImVBRCiISEBCGTycTQoUOFEEJkZ2eLzp07C0tLSzFmzBixcuVKsWDBAkMNqamphmWHDx9u+J0uWbJELFq0SDz33HNi6dKlQgghoqOjhUwmE+3btxfLli0Tc+fOFc2aNRN+fn4CgIiNjTWsqyx3jRW1PXgHV0latWolOnbsWOrnS5cuFQDEmTNnDG1jxowRAMSAAQPEokWLxKJFi8SAAQMEADFmzJhi6/jzzz+Fk5OTUCqV4vXXXxerVq0Sq1atEm+88YawtbUVffv2fWiNZXHmzBkhl8tFp06dxMqVK8XHH38sFApFsXUX/b8VFhZmaEtLSxPu7u5CJpOJd955R3z11Vdi7NixQiqVirZt2wqNRlNse88//7yQy+UiPT290rUTVQaDENG/VEUQEqLw9ml/f3+hVCqFra2taN++vZg8ebK4ffu2Ub8dO3aI7t27C6VSKezs7ES3bt3EDz/8YLSdB4PQ1q1bRd++fYWrq6uQyWSiUaNGYuzYsSIxMdHQp6QgJIQQmzdvFp06dRJyuVw4OTmJYcOGGaYDeNR+VUUQEkKIUaNGCalUKq5duyaEECIzM1NMnTpV+Pr6CplMJpydnUX37t3FggULhFarNSxXUFAg5s+fL1q1aiVkMplwcXER/fv3NwoYO3bsEB06dBAKhUL4+PiIefPmGWY3Lm8QunDhQonTIzzozJkzAoCYNm1aqX1u3LghAIhJkyYZ2nQ6nViyZInw8/MTCoVCKBQK4efnJ7788ktDmP2327dvi0mTJokWLVoIhUIhrK2thb+/v5g9e7bRbNCVceTIEdG9e3ehUCiEi4uLGD9+vFCr1UZ9SgpCQghx69Yt8eqrr4omTZoImUwmPDw8xOuvv24UZotkZGQIhUIhnnvuuSqpm6gyJEI85Bw8EZGZWrFiBSZPnozr168XmzGZiOoPjhEiIirBwYMH8c477zAEEdVzPCNEREREZotnhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmy+wesaHX63H79m3Y2to+9AnfREREVHsIIZCZmQlPT89iz3isDLMLQrdv34a3t7epyyAiIqIKiI+PR8OGDatsfWYXhGxtbQEU/iLt7OxMXA0RERGVhVqthre3t+F7vKqYXRAquhxmZ2fHIERERFTHVPWwFg6WJiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmy6RB6PDhwxg4cCA8PT0hkUiwffv2Ry5z6NAhdO7cGXK5HL6+vli3bl2110lERET1k0mDUHZ2Nvz8/LB8+fIy9Y+NjcVTTz2F3r17IzIyEu+++y7GjBmDvXv3VnOlREREVB+Z9KGr/fv3R//+/cvcf9WqVWjSpAm++OILAEDr1q1x9OhRLFq0CCEhIdVVJhEREdVTderp8ydOnEBwcLBRW0hICN59991yr2vfpSTYqLKrqDKqjeyUVgho0gBSi6p9UjEREdUfdSoIJSUlwc3NzajNzc0NarUaubm5UCqVxZbRaDTQaDSG92q1GgAwafN5WMitq7dgMrmFQ/zwXOeGpi6DiIhqqToVhCoiPDwcM2fOLNbeydsBVkobE1RENeHGnWykZWmRpM4zdSlERFSL1akg5O7ujuTkZKO25ORk2NnZlXg2CACmTp2K0NBQw3u1Wg1vb298NyYAdnZ21Vovmc7krefx4+lbpi6DiIhquToVhAIDA7Fr1y6jtn379iEwMLDUZeRyOeRyeXWXRkRERHWQSW+fz8rKQmRkJCIjIwEU3h4fGRmJuLg4AIVnc0aMGGHo/+abbyImJgaTJ09GdHQ0VqxYgR9//BGTJk0yRflUB2Tk5kOdl2/qMoiIqJYyaRA6ffo0OnXqhE6dOgEAQkND0alTJ0yfPh0AkJiYaAhFANCkSRPs3LkT+/btg5+fH7744gusWbOGt85Tqb76IwZdZu3HtZQsU5dCRES1kEQIIUxdRE1Sq9Wwt7dHRkYGxwjVYzvO38YHW85DU6AHAKx6pTP6tfMwcVVERFRR1fX9zWeNUb30jJ8nomf1Q5fGjqYuhYiIajEGIaq3JBJOpEhERA9Xp+4aI6qo66nZOBCdjHvZ+biXo0V6Tj7u5miRp9XhlcDG6NyIZ46IiMwRgxCZhfl7L5f6mTovH2tGdq3BaoiIqLZgEKJ6bUB7D1xLzYKNzBKONlZwtJbdf1nhdkYe9l1KNgyoJiIi88MgRPXaq481wauPNSnxs23nbmHfpeQSP/s3vV4gPTcfaVkapGVqkJqlQWqmBmlZWiisLDD28WZQyqRVWToREdUABiEye9maAvwVcwfJmRqkqPOQmqVBWqYWaYawo8HdbC0K9KXPNOHrqsLTHTxrsGoiIqoKDEJk9s7GpWPo13+Wqa+90grOKhlcbOVwVslx9uY93M7IQ45WV81VEhFRdWAQIrPl19ABjtZWyMvXw91eAVdbOVztFHBRyeFsK4OzSl748/33DWzkkFkazzjx6rpTuJ1h/IR7nV7gTrYGzjZyWFjwFn4iotqMQYjMVlMXFc588iQkksrPOfTN0Vh8/+dNJKvzkJqpgV4APZs747vXAqqoWiIiqg4MQmTWKnvGRiUv/CMUnZRZ7LP/3cqo1LqJiKj6MQgRVcIHIS3R0t0WdkoruNsp4GYnR65WV+YxR0REZFoMQkSV4O1kjfG9fY3arqcWPuk+L1+HJfuvon97d7Rws63WOoQQuJutRZI6D8nqPCRlaAp/zshDkjoP7bzs8EFIq2qtgYioLmIQIqpilvcvt2kK9Fi0/wpO37xbqbFCBTo9kjM1uJ2ei6SMoqCT90/oUechOUMDra70iSH/uJKK1x5rCicbWYXrICKqjxiEiKpYIydrvNqjCf6KvYO/b6uRmVfw0P7aAj0SM3KRcC8Xt+7l4lZ6Lm7dy0HCvVwkpOciMSMPuofMYfQgZ5UMbnaKwst09oX/XbjvCgCgQM8ZtImI/o1BiKiKSSQSTB/YBvsvJWPMt6chhEBSRh5u3snGzbs5iLuTg/h7Obh1rzD8JGfmQTwi51hJJXC3V8DDTnk/4MgLA8/9sON2//Xv2/sBYPH+KyhjjiIiMjsMQkTV7PytDPxfeMRD+8gtLeDlqERDR2t4OSjR0LHwVfizNVxs5ZByTiIioirHIERUTXycrQ0/Sy0k8HJQonEDazRyKnw1dLS+H36UaGAjq/RcRmWRl69DQnrhmSh1Xj4eb+ECO4VVtW+XiKi2YhAiqia+rrY4MbUPtAV6eDooYSUtftmqJoUsOox7OflGba891gTTnm5jooqIiEyPQYioGnnYK01dAtzsFEjMyDOEIBuZFHIrKe5ma5GaqTFxdUREpiUR4lHDNOsXtVoNe3t7ZGRkwM7OztTlEFW7uDs5iE5Sw/P+2CN7pRXWHruBT3+7BDc7Odztlbh1NwdSCwl+eqs7vJ2sH71SIqIaVl3f3zwjRFTPNWpgjUYNjMNN0aNBktUaJKv/OSt0Nu4egxARmRUGISIz9FQHD9zL0UJqIUFDR2ssP3gNFxIy8MflVMgtpejXzt3UJRIR1QgGISIzZCO3xNigZob3G/66CQD4+VwCfj6XgD8+6IXGDWxKXFavF9Dq9FBYSWukViKi6sQgRER4qVsjqHPzEZWYCa1Oj3s5+bCQ5ODGnWzcuJODm2nZhp/j7uYAAvjutW4IaNrA1KUTEVUKgxARYUB7Dwxo74Hu4RG4nZGH51Yce+Rs1OdvpTMIEVGdxyBERAZejkrczsiDXgAyqQW8nZRo4myDxg1s4NPAGj7ONlh//Ab2R6WYulQioirBIEREBmtGdEVUkhoNHZXwsFeW+FiPbecSTFAZEVH1YBAiIgN7ayv8Hy93EZEZMe2c/0REREQmxCBEREREZotBiIiIiMwWgxARERGZLQYhIiIiMlsMQkRERGS2GISIqMLUefk4deMurqdmmboUIqIK4TxCRFQhi/dfxZxd0QAAmaUF/pr6BBxtZCauioiofHhGiIjKxVklBwDkaHWGNm2BHvuikpGYkWuqsoiIKkQihHjEoxXrF7VaDXt7e2RkZMDOzs7U5RDVORk5+fjjaircbOVo5W6Hx+cfREZuPgDAw16B41P6QCL559EcBTo9LKX8NxcRVU51fX/z0hgRlYu9tRWe8fM0vO/WxAn7LiUDABIz8rDj/G1EJWbiUqIaUYlqpOdosfSlTujXzsNUJRMRlYpBiIgqZfWILkjMyEVg+AEAwMRNkcX6HIxOhb1ShqhENRRWUrzUzdvorBERkakwCBFRpbnZKtCpkQOiEzPRysMWbTzs0MbTDn/G3MWv529j8+l4bD4db+jfzssOHRo6mK5gIqL7GISIqNIsLCTYNq4H9HoBC4t/zvTILaX49fxtAICXgxJ3sjXIy9cjK6/AVKUSERlhECKiKvNgCAKA5zt7oUtjRzhay2BvbYWQRYdxOTnTRNURERXHIERE1UYikcDH2cbUZRARlYpBiIioCmVrCnA9NQtXk7NwNSUL11IyAUiwcKgf7BRWpi6PiP6FQYiIqAIycvNx7X7Q+Sf0ZCEhveRJJY9dTUP/9pxCgKi2YRAionolV6tDVJIaDkorNHVRVXp9WZoCXE7KRHSSGleTC8PO1ZRMJKs1pS7jrJKjuasKzd1UOBCdglv3cqG7P3dteo4WKrklJ5kkqiUYhIioxu2LSoYA0MPXuVLrUefl4+8ENf6+nYG/b6txMSED11OzoBeATGqBE1P7wMlGhlv3cnEpUY34uzkIaesObyfrYuvS6QXi7uYgOlGNqKRMRCeqEZ2Uibi7OaVu38NeAV9XFXxdVWjuaovmbir4uqiMnrl2OekEbt3LxYK9lzH9l79xN1uLFm4q7Jn4OAAgIT33/pmlwoClKdDj46daw9VWYVhHvk4PKwYnomrBIERENW7tsRv4/s+bOP3Jk7BXlm3czJ0sDS7evh96EtS4eDsDN++UHlK0Oj2G//ck4u/lIPOB2/XPxadjzqD2iE4qnPk6OikTUUmZuJKUidx8XYnrcrdToKW7LVq6294PPSo0c1WVacyPraLwr9kbD9R6JTkLTy76AwnpucjL1xdbJiY1G92bNSgMSKlZiL+bg+7NnPH9mIBHbo+IyofPGiOiGrNk/1V89+cNpGVpARROrNi3jTveeaK5Ub+M3HxcuJWB87fSERmfjosJGUjMyCtxnV4OSrTzskM7T3u087JHW087PLfyOG7d+2esjpVUAkdrGVIyNbCQAPpS/tZTWFmgpZstWrnboZXH/f+62xqd4SmvaylZ2HMxEV6OSjRyssbzK08YfS6TWqCpiw2auaqw83+JD12XX0N7DO7ijVf+r3GF6yGqq6rr+5tBiIhqXIcZe6G+f5ZGaSXFhtcD8L/4dJy/lYHz8emIScsutoxEAjRxtkE7z8KwUxR6HKyLh5SjV9Nw5FoqWrjaoo2nHZq5qHD65l28vPovQ5+Gjkq0crdDa49/go9PAxtILar30R+HLqcgKjHTcEnN21FpGC90MDoFc3dHw9lWBl+Xws9d7RQY+90Zw/K+rirsDw2q1hqJaiMGoSrCIERkenv/TsLB6BRsOhVfap/GDazRoaED/Brao0NDB7TxtINKXvGr+UII/BlzF1ZSCVq429apW9kjopKxPyoZP5yMh08Da3w9oguuJmchIzcfT/t51Kl9IaooBqEqwiBEVDvkanUInBuB9Jx8OKtk8GvoUBh8vO3h19ChUpej6qO/Yu5g6Nd/FmsPaeuGNx5vhs6NHPggW6rXquv7m4OlicgklDIpDk/uDXVuPrwclPwSf4TGDWxgJZUgXydgK7dEpqbw0uLev5Ox9+9krH+1G4JauJi4SqK6h0GIiEzGTmHFyzpl5G6vwF8fBUNboIebnRwHolMwbsNZaHV6CAHcvj+RY16+DjGp2YhJy0JbT3s04SNOiB6Kl8aIiOoonV7gze/PYN+lZLRwUyFfJ3DzTrbhrrgmzjY4+H4vk9ZIVFV4aYyIiIxILSSwlkkBFM5NVMRaJkWOVofUTA2EEEjL0iIlMw8t3Ww5ozXRvzAIERHVYW/38YW7nQLu9go0d7VFCzcVcrQ69FpwCNnaAnSetQ/3cvIBACMCG+P1nk3R0LHkMVlZmgLYyKQcr0VmhZfGiIjqmbvZWnSbvR8FpcwcOevZtmjpbocryZm4mpyJqylZuJKcibQsLULaumHBYD/YcuwW1TL19vb55cuXY/78+UhKSoKfnx+WLl2Kbt26ldp/8eLFWLlyJeLi4uDs7IwXXngB4eHhUCgUpS7zIAYhIjIHf8bcwe30XLRws0WyOg+vrT9druXfeLwpxvfyhb01AxHVDvUyCG3evBkjRozAqlWrEBAQgMWLF2PLli24fPkyXF1di/XfuHEjXn31VXzzzTfo3r07rly5glGjRuHFF1/EwoULy7RNBiEiMkd3sjTYfDoen++5DKDw0SQt3FRo7maL5q4qOKvkGL3uVLHlPnmqNQZ18oKzSl7TJRMZqZdBKCAgAF27dsWyZcsAAHq9Ht7e3nj77bcxZcqUYv0nTJiAqKgoREREGNree+89/PXXXzh69GiZtskgRETmSgiBW/dy4WQjg00Js3TfTs/Fiet38N6W88U+4zxFZGrV9f1tstsHtFotzpw5g+Dg4H+KsbBAcHAwTpw4UeIy3bt3x5kzZ3Dy5EkAQExMDHbt2oUBAwaUuh2NRgO1Wm30IiIyRxKJBN5O1iWGIADwdFDief+GOPxBb0zt38ros8tJD/+7M1+nx9XkTJy6cRf5On2V1UxU3Ux211haWhp0Oh3c3NyM2t3c3BAdHV3iMi+//DLS0tLw2GOPQQiBgoICvPnmm/joo49K3U54eDhmzpxZpbUTEdVnjRpYY2xQMwxo74HPdl7C3r+TsXj/VWz8Kw4/jg1EpqYAV5MzcSU56/6A6yzEpGUhX1d4gWHa023w2mNNTLwXRGVTp26fP3ToEObMmYMVK1YgICAA165dw8SJEzFr1ixMmzatxGWmTp2K0NBQw3u1Wg1vb++aKpmIqM7ydrKGT4PCmalztDrcuJODbnMiHrEUcOm2GvF3c+DtZG3ULoRAapYG15KzcDUlCymZeRjs7w2fMsx+naMtgLWsTn1lUR1hsjFCWq0W1tbW2Lp1KwYNGmRoHzlyJNLT0/HLL78UW6Znz574v//7P8yfP9/Q9v333+ONN95AVlYWLCwefaWPY4SIiMpOnZePw1dS8emvl5CSqQEAKK2k8HVVoYVb4bxFLdxs0dxNhQ1/xWHloeuGZcf3bgZnlRxXU7IMt+mn35/T6EFPdfDAhyGt4O2kxO2MPFxLybr/ysS1lCzDcgPau2OwvzeupWQhN1+Hkd19YK+0gk4vEH83B7Fp2WjqYoPGDfhYkfqo3s0sLZPJ4O/vj4iICEMQ0uv1iIiIwIQJE0pcJicnp1jYkUoLZ1U1s+mQiIhqhJ3CCk938MTjLVxw4VYGvB2t0dBRCQuL4pMutvU0/nJafvB6sT4SCdDYyRo37uQY2nb+LxE7/5domBG7NLsuJGHXhSTD+4X7rqClmy1i07KhvT8uyVklx8mPniixPqKSmPQ8Y2hoKEaOHIkuXbqgW7duWLx4MbKzszF69GgAwIgRI+Dl5YXw8HAAwMCBA7Fw4UJ06tTJcGls2rRpGDhwoCEQERFR1bNTWKGHr/ND+zzdwRPdmznjt//dxtzd0XCzU6C5qwrN75818nVVoZmLCgorKdR5+Vh+8Bq++iPGsHyOVgdLCwl8nG3Q3FUF3/svhZUU4zechYVEgqYuNohOyjQsczm58GeZ1AJanR5pWRqsPhKDns1d0MaTZ/3p0Uw+oeKyZcsMEyp27NgRX375JQICAgAAvXr1go+PD9atWwcAKCgowOzZs/Hdd98hISEBLi4uGDhwIGbPng0HB4cybY+XxoiIqp8QokyP6sjIyceui4lwtLaCr6sKjRvYwKqE56Hl6/SwkEggtZAg/m4OfjgZBycbGZq5quDrooKdwgp+n/5u6N/W0w473+lZpftEplUv5xEyBQYhIqL66aNtF3D8Whpu3MmBjUyKx5o743pqNtJztJg/2A+9WxafqJfqjno3RoiIiKgqzflPe1y6rcaAL48gW6vD3r+TDZ/9GnkbcksLXE/NxvWULFxPzUITZxvMfKYtHzJr5nhGiIiI6g0hBL74/Qru5mjh66LC6Zt3jQZY/9vxKX3g6aCswQqponhGiIiI6BEkEgneD2lpeO9ur8CuC0mwkBTOi9TMpXAA9jdHY1GgF8jSFOBiQgbi7+bAv7EjXO3K9gBvqj94RoiIiOq1u9laWMukUFj9c3dx62l7kJtvfKv+Y77O+H5M4c06pQ32zsvXGSaLfHB9mgId8vL1sFdaVdNeEM8IERERVYCTjaxYW0NHJa6mZAEonCAyN1+H87fSMfKbk7iWkoWE9Fy8G9wcHvYKwwSP11OzEXf3n/mP3ni8qWG80c27ORACCGnrhsYNbBCTmgVXOwU+e7Yd5zSq5XhGiIiIzM7dbC1i07LQxFmF2LQsPL+y5Id9V9ZXw/0hBBCTloXrKdlIVudhWEAjXoarAN4+X0UYhIiI6EF6vcCi/VeQrdHB11WFm3ez8dUfMfC0V6DZ/UkgiyaDbOZqg0X7ruLWvZz771Vo5mKDtCwtvj58HY7WMjRzUWHL6XhkP2SWbACYNagdhv9fYwCAtkCPuLs5aGAjg2MJZ7CIQajKMAgREdGjlHVCyNK8uu4UDkSnwEYmRVOXwrCkKdBj98V/7mBr7qpC4wbWiEnNxs27OdDpBVRyS/z50RNQyTly5d84RoiIiKiGVHZuoTUjuuBujhYNbGRG6yp6BMhXf8QUPoz2/jilIlmaAqRlaqCwtED8vVxcT8lCTFoWYlKzkZlXgLFBTQEAsWnZuJ6ajdi0bMSkZiHuTg76tXPH/MF+larbHPGMEBERUQ26nZ6LxfuvwFpmiWYuNmjqokJTFxs8ufAwsjQFaOioRLI6D/m68n09q+SWuDgzpJqqNj2eESIiIqoHPB2U+PyF4mduHKytkKUpwK17uQAAhZUFmjgXhqTIuHQkpBe2O6vkaOpig6bONmjqYgOlzBLTtl9Ebr4Oo9eeRP/2HvB1VSE2NRs5+Tr8p5MXL7U9BH8zREREtcCqV/xxLu4eGjcoDDie9kqjW+9vp+dCpbCEncJ4rqIUdR6mbb8InV7g4OVUHLycavR5foEerz7WpEb2oS5iECIiIqoF2nnZo52Xfamfl/YoEFc7BT5/vgNOxNzBtnMJAAB3OwXydXrcydbiQkIGzty8i86NHPlctRJwjBAREVE9kaMtgBCAjdwSn2y/gO//jDN89uPYQHRr4mTC6iqnur6/LapsTURERGRS1jJL2NwfD/SYrwtUcktI719eO3PzHlIzNaYsr1ZiECIiIqqH+rVzx4UZffF/TQvPAs3bE40e8w4gLYth6EEMQkRERPWURCKBX0MHw3ttgR7n4tKRl//wWa/NCccIERER1XMZOfnot+QwEjPyAADeTkocer+34bJZXcAxQkRERFQh9tZWaOlua3gffzcXuf86K6Qt0ONaSiYuJmRArzefcyS8fZ6IiMgMrBnRBTfv5uCJL/4AAPx89hYS0nNxPaXwMR1FzzsDgGUvd0J7L3tcT82Cq63iobf113UMQkRERGbAUmoBrwfmIpr+y9+l9p2w8ZzhZwsJ8NNb3ZGek4/4ezno3qwBmrmo6s2cRAxCREREZkJhJcWzHT1x+sY9NHWxQTMXleF5Z81cVPjhZByWRFwFAMgtLaAp0EMvgP+sOG60HpmlBab2b4XRPZogMy8fManZiEnLgo3MEn3bupti1yqMg6WJiIgIAJCv0+Pv22o0sJHBy0GJl1b/ib9i78JKKoHCUopMTYFRfxdbebG5iaY/3QZ9WrnCx9mmSmurru9vBiEiIiIqUYFOj5RMDdzsFJBaSBCdpMa2swn46nCMUb9/ByIbmRQnPw42TO5YFfj0eSIiIqpRllILo2ectXK3wwchKrRws4VEAjRzUaGJiw3sFFYI3x2Fb47GIl8nkK3VQZ2XX6VBqLrwjBARERFVCSEEWn6yB1qdHr+M74G2nnawlFbNTD08I0RERES12oN3kj27/BgAIKiFC4Z29caA9h6mKuuhOKEiERERVRlfV5XR+z+upGLchrN4+4dzOB+fbpqiHoJnhIiIiKjK/DyuO5Iy8vBX7B38fDYBf8XeBQD8ev42pBJg8YudTFyhMQYhIiIiqjIKKyl8nG3g42yDF/y9sWT/FZyIuYNTN+4hX1f7hiXz0hgRERFVC6mFBKF9W+LpDp6mLqVUDEJERERkthiEiIiIqEZoCvS4m601dRlGGISIiIioRuyPSkbnWfuw4a+bSMvSPHqBGsAgRERERNWqmYvxLfUfb7uILp/txy+RCUhR55moqkKcWZqIiIiq3e30XPx89hYW/H6l2Ge2cks80doVi4Z2NJqU8UHV9f3NM0JERERU7TwdlJjQpzn++ugJjHmsidFnmZoCbI+8DXVuQSlLVx/OI0REREQ1xs1OgU+eboMJfXyRlqXBiZi7mLb9IgAgUZ0LG7m0yp5PVhY8I0REREQ1zsFaBl9XW7zY1dvQ1m/xEYQsPoyzcfeQq9XVSB08I0REREQmY2khQRNnG8SmZQMArqdm47kVxw2fe9orENTSBWMC3Ktn+9WyViIiIqIykEgk2D2xJ27eycHw//6FlEzj2+pvZ+Thh5Px8LSunnu7GISIiIjIpBRWUrR0t8WJqU+gQK/Hqdh7SFLnYc/FJMSkZiEmLRtanb5ats0gRERERLWC1EICqYUUjzV3BgC84N8Qk7eeR8z9y2bVgYOliYiIyGwxCBEREVGtt/lkfLWsl0GIiIiIai2V3AoAkKSunmeTMQgRERFRrTU2qCmm9m8FS4uSH71RWXzWGBEREdV6fNYYERERURVjECIiIiKzxSBEREREZotBiIiIiMwWgxARERGZLQYhIiIiMlsMQkRERGS2GISIiIjIbDEIERERkdliECIiIiKzxSBEREREZsvkQWj58uXw8fGBQqFAQEAATp48+dD+6enpGD9+PDw8PCCXy9GiRQvs2rWrhqolIiKi+sTSlBvfvHkzQkNDsWrVKgQEBGDx4sUICQnB5cuX4erqWqy/VqvFk08+CVdXV2zduhVeXl64efMmHBwcar54IiIiqvNM+vT5gIAAdO3aFcuWLQMA6PV6eHt74+2338aUKVOK9V+1ahXmz5+P6OhoWFlZVWibfPo8ERFR3VPvnj6v1Wpx5swZBAcH/1OMhQWCg4Nx4sSJEpfZsWMHAgMDMX78eLi5uaFdu3aYM2cOdDpdTZVNRERE9UiFLo3pdDqsW7cOERERSElJgV6vN/r8wIEDj1xHWloadDod3NzcjNrd3NwQHR1d4jIxMTE4cOAAhg0bhl27duHatWsYN24c8vPzERYWVuIyGo0GGo3G8F6tVj+yNiIiIjIPFQpCEydOxLp16/DUU0+hXbt2kEgkVV1XifR6PVxdXfH1119DKpXC398fCQkJmD9/fqlBKDw8HDNnzqyR+oiIiKhuqVAQ2rRpE3788UcMGDCgwht2dnaGVCpFcnKyUXtycjLc3d1LXMbDwwNWVlaQSqWGttatWyMpKQlarRYymazYMlOnTkVoaKjhvVqthre3d4XrJiIiovqjQmOEZDIZfH19K7VhmUwGf39/REREGNr0ej0iIiIQGBhY4jI9evTAtWvXjC7FXblyBR4eHiWGIACQy+Wws7MzehEREREBFQxC7733HpYsWYLK3nAWGhqK1atXY/369YiKisJbb72F7OxsjB49GgAwYsQITJ061dD/rbfewt27dzFx4kRcuXIFO3fuxJw5czB+/PhK1UFERETmqUKXxo4ePYqDBw9i9+7daNu2bbFb2X/++ecyrWfo0KFITU3F9OnTkZSUhI4dO2LPnj2GAdRxcXGwsPgnq3l7e2Pv3r2YNGkSOnToAC8vL0ycOBEffvhhRXaDiIiIzFyF5hEqOmNTmrVr11a4oOrGeYSIiIjqnur6/q7QGaHaHHSIiIiIyqpSj9hITU3F5cuXAQAtW7aEi4tLlRRFREREVBMqNFg6Ozsbr776Kjw8PPD444/j8ccfh6enJ1577TXk5ORUdY1ERERE1aJCQSg0NBR//PEHfv31V6SnpyM9PR2//PIL/vjjD7z33ntVXSMRERFRtajQYGlnZ2ds3boVvXr1Mmo/ePAghgwZgtTU1Kqqr8pxsDQREVHdU6seupqTk1PsGWEA4OrqyktjREREVGdUKAgFBgYiLCwMeXl5hrbc3FzMnDmz1FmhiYiIiGqbCt01tmTJEoSEhKBhw4bw8/MDAJw/fx4KhQJ79+6t0gKJiIiIqkuFxggBhZfHNmzYgOjoaACFDz8dNmwYlEpllRZY1ThGiIiIqO6pVRMqAoC1tTVef/31KiuEiIiIqKaVOQjt2LED/fv3h5WVFXbs2PHQvs8880ylCyMiIiKqbmW+NGZhYYGkpCS4uroaPQi12AolEuh0uiorsKrx0hgREVHdY/JLY3q9vsSfiYiIiOqqCt0+X5L09PSqWhURERFRjahQEJo3bx42b95seD948GA4OTnBy8sL58+fr7LiiIiIiKpThYLQqlWr4O3tDQDYt28f9u/fjz179qB///744IMPqrRAIiIioupSodvnk5KSDEHot99+w5AhQ9C3b1/4+PggICCgSgskIiIiqi4VOiPk6OiI+Ph4AMCePXsQHBwMABBC1Oo7xoiIiIgeVKEzQs899xxefvllNG/eHHfu3EH//v0BAOfOnYOvr2+VFkhERERUXSoUhBYtWgQfHx/Ex8fj888/h0qlAgAkJiZi3LhxVVogERERUXWp8LPG6ipOqEhERFT3mHxCRT5ig4iIiOobPmKDiIiIaj2TnxHiIzaIiIiovqmyR2wQERER1TUVCkLvvPMOvvzyy2Lty5Ytw7vvvlvZmoiIiIhqRIWC0E8//YQePXoUa+/evTu2bt1a6aKIiIiIakKFgtCdO3dgb29frN3Ozg5paWmVLoqIiIioJlQoCPn6+mLPnj3F2nfv3o2mTZtWuigiIiKimlChmaVDQ0MxYcIEpKamok+fPgCAiIgIfPHFF1i8eHFV1kdERERUbSoUhF599VVoNBrMnj0bs2bNAgD4+Phg5cqVGDFiRJUWSERERFRdKv2IjdTUVCiVSsPzxmo7TqhIRERU91TX93eF5xEqKCjA/v378fPPP6MoS92+fRtZWVlVVhwRERFRdarQpbGbN2+iX79+iIuLg0ajwZNPPglbW1vMmzcPGo0Gq1atquo6iYiIiKpchc4ITZw4EV26dMG9e/egVCoN7f/5z38QERFRZcURERERVacKnRE6cuQIjh8/DplMZtTu4+ODhISEKimMiIiIqLpV6IyQXq8v8Qnzt27dgq2tbaWLIiIiIqoJFQpCffv2NZovSCKRICsrC2FhYRgwYEBV1UZERERUrSp0+3x8fDz69esHIQSuXr2KLl264OrVq3B2dsbhw4fh6upaHbVWCd4+T0REVPdU1/d3hecRKigowObNm3H+/HlkZWWhc+fOGDZsmNHg6dqIQYiIiKjuqTVBKD8/H61atcJvv/2G1q1bV1khNYVBiIiIqO6pNRMqWllZIS8vr8oKICIiIjKVCg2WHj9+PObNm4eCgoKqroeIiIioxlRoHqFTp04hIiICv//+O9q3bw8bGxujz3/++ecqKY6IiIioOlUoCDk4OOD555+v6lqIiIiIalS5gpBer8f8+fNx5coVaLVa9OnTBzNmzKj1d4oRERERlaRcY4Rmz56Njz76CCqVCl5eXvjyyy8xfvz46qqNiIiIqFqVKwh9++23WLFiBfbu3Yvt27fj119/xYYNG6DX66urPiIiIqJqU64gFBcXZ/QIjeDgYEgkEty+fbvKCyMiIiKqbuUKQgUFBVAoFEZtVlZWyM/Pr9KiiIiIiGpCuQZLCyEwatQoyOVyQ1teXh7efPNNo1voefs8ERER1QXlCkIjR44s1vbKK69UWTFERERENalcQWjt2rXVVQcRERFRjavQIzaIiIiI6gMGISIiIjJbDEJERERkthiEiIiIyGwxCBEREZHZYhAiIiIis8UgRERERGarVgSh5cuXw8fHBwqFAgEBATh58mSZltu0aRMkEgkGDRpUvQUSERFRvWTyILR582aEhoYiLCwMZ8+ehZ+fH0JCQpCSkvLQ5W7cuIH3338fPXv2rKFKiYiIqL4xeRBauHAhXn/9dYwePRpt2rTBqlWrYG1tjW+++abUZXQ6HYYNG4aZM2eiadOmNVgtERER1ScmDUJarRZnzpxBcHCwoc3CwgLBwcE4ceJEqct9+umncHV1xWuvvfbIbWg0GqjVaqMXEREREWDiIJSWlgadTgc3Nzejdjc3NyQlJZW4zNGjR/Hf//4Xq1evLtM2wsPDYW9vb3h5e3tXum4iIiKqH0x+aaw8MjMzMXz4cKxevRrOzs5lWmbq1KnIyMgwvOLj46u5SiIiIqoryvX0+arm7OwMqVSK5ORko/bk5GS4u7sX63/9+nXcuHEDAwcONLTp9XoAgKWlJS5fvoxmzZoZLSOXyyGXy6uheiIiIqrrTHpGSCaTwd/fHxEREYY2vV6PiIgIBAYGFuvfqlUrXLhwAZGRkYbXM888g969eyMyMpKXvYiIiKhcTHpGCABCQ0MxcuRIdOnSBd26dcPixYuRnZ2N0aNHAwBGjBgBLy8vhIeHQ6FQoF27dkbLOzg4AECxdiIiIqJHMXkQGjp0KFJTUzF9+nQkJSWhY8eO2LNnj2EAdVxcHCws6tRQJiIiIqojJEIIYeoiapJarYa9vT0yMjJgZ2dn6nKIiIioDKrr+5unWoiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrNVK4LQ8uXL4ePjA4VCgYCAAJw8ebLUvqtXr0bPnj3h6OgIR0dHBAcHP7Q/ERERUWlMHoQ2b96M0NBQhIWF4ezZs/Dz80NISAhSUlJK7H/o0CG89NJLOHjwIE6cOAFvb2/07dsXCQkJNVw5ERER1XUSIYQwZQEBAQHo2rUrli1bBgDQ6/Xw9vbG22+/jSlTpjxyeZ1OB0dHRyxbtgwjRox4ZH+1Wg17e3tkZGTAzs6u0vUTERFR9auu72+TnhHSarU4c+YMgoODDW0WFhYIDg7GiRMnyrSOnJwc5Ofnw8nJqcTPNRoN1Gq10YuIiIgIMHEQSktLg06ng5ubm1G7m5sbkpKSyrSODz/8EJ6enkZh6kHh4eGwt7c3vLy9vStdNxEREdUPJh8jVBlz587Fpk2bsG3bNigUihL7TJ06FRkZGYZXfHx8DVdJREREtZWlKTfu7OwMqVSK5ORko/bk5GS4u7s/dNkFCxZg7ty52L9/Pzp06FBqP7lcDrlcXiX1EhERUf1i0jNCMpkM/v7+iIiIMLTp9XpEREQgMDCw1OU+//xzzJo1C3v27EGXLl1qolQiIiKqh0x6RggAQkNDMXLkSHTp0gXdunXD4sWLkZ2djdGjRwMARowYAS8vL4SHhwMA5s2bh+nTp2Pjxo3w8fExjCVSqVRQqVQm2w8iIiKqe0wehIYOHYrU1FRMnz4dSUlJ6NixI/bs2WMYQB0XFwcLi39OXK1cuRJarRYvvPCC0XrCwsIwY8aMmiydiIiI6jiTzyNU0ziPEBERUd1TL+cRIiIiIjIlBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmi0GIiIiIzBaDEBEREZktBiEiIiIyWwxCREREZLYYhIiIiMhsMQgRERGR2WIQIiIiIrPFIERERERmy9LUBdRGQggUFBRAp9OZuhSiWkUqlcLS0hISicTUpRARVQkGoX/RarVITExETk6OqUshqpWsra3h4eEBmUxm6lKIiCqNQegBer0esbGxkEql8PT0hEwm4798ie4TQkCr1SI1NRWxsbFo3rw5LCx4dZ2I6jYGoQdotVro9Xp4e3vD2tra1OUQ1TpKpRJWVla4efMmtFotFAqFqUsiIqoU/nOuBPxXLlHp+OeDiOoT/o1GREREZotBiCpFIpFg+/btVd63rjt06BAkEgnS09MBAOvWrYODg4NJayIiouIYhOqJUaNGQSKRQCKRQCaTwdfXF59++ikKCgqqdbuJiYno379/lfetDB8fH8PvwtraGu3bt8eaNWuqfbtERFT3MAjVI/369UNiYiKuXr2K9957DzNmzMD8+fNL7KvVaqtkm+7u7pDL5VXet7I+/fRTJCYm4uLFi3jllVfw+uuvY/fu3TWy7dqiqo4xEVF9xiBUj8jlcri7u6Nx48Z46623EBwcjB07dgAoPGM0aNAgzJ49G56enmjZsiUAID4+HkOGDIGDgwOcnJzw7LPP4saNG0br/eabb9C2bVvI5XJ4eHhgwoQJhs8evNyl1WoxYcIEeHh4QKFQoHHjxggPDy+xLwBcuHABffr0gVKpRIMGDfDGG28gKyvL8HlRzQsWLICHhwcaNGiA8ePHIz8//5G/C1tbW7i7u6Np06b48MMP4eTkhH379hk+T09Px5gxY+Di4gI7Ozv06dMH58+fN1rHr7/+iq5du0KhUMDZ2Rn/+c9/DJ9999136NKli2E7L7/8MlJSUh5Z18PcunULL730EpycnGBjY4MuXbrgr7/+MvpdPOjdd99Fr169DO979eqFCRMm4N1334WzszNCQkLw8ssvY+jQoUbL5efnw9nZGd9++y2AwmkjwsPD0aRJEyiVSvj5+WHr1q2V2hciorqCt88/ghACufmmmWFaaSWt1DxGSqUSd+7cMbyPiIiAnZ2dIRDk5+cjJCQEgYGBOHLkCCwtLfHZZ5+hX79++N///geZTIaVK1ciNDQUc+fORf/+/ZGRkYFjx46VuL0vv/wSO3bswI8//ohGjRohPj4e8fHxJfbNzs42bPvUqVNISUnBmDFjMGHCBKxbt87Q7+DBg/Dw8MDBgwdx7do1DB06FB07dsTrr79ept+BXq/Htm3bcO/ePaMJAAcPHgylUondu3fD3t4eX331FZ544glcuXIFTk5O2LlzJ/7zn//g448/xrfffgutVotdu3YZls/Pz8esWbPQsmVLpKSkIDQ0FKNGjTLqUx5ZWVkICgqCl5cXduzYAXd3d5w9exZ6vb5c61m/fj3eeustwzG6du0aBg8ejKysLKhUKgDA3r17kZOTYwh24eHh+P7777Fq1So0b94chw8fxiuvvAIXFxcEBQVVaH+IiOoKBqFHyM3Xoc30vSbZ9qVPQ2AtK/8hEkIgIiICe/fuxdtvv21ot7GxwZo1awyB4Pvvv4der8eaNWsMgWvt2rVwcHDAoUOH0LdvX3z22Wd47733MHHiRMN6unbtWuJ24+Li0Lx5czz22GOQSCRo3LhxqTVu3LgReXl5+Pbbb2FjYwMAWLZsGQYOHIh58+bBzc0NAODo6Ihly5ZBKpWiVatWeOqppxAREfHIIPThhx/ik08+gUajQUFBAZycnDBmzBgAwNGjR3Hy5EmkpKQYLtUtWLAA27dvx9atW/HGG29g9uzZePHFFzFz5kzDOv38/Aw/v/rqq4afmzZtii+//BJdu3Y1ChzlsXHjRqSmpuLUqVNwcnICAPj6+pZ7Pc2bN8fnn39ueN+sWTPY2Nhg27ZtGD58uGFbzzzzDGxtbaHRaDBnzhzs378fgYGBhv05evQovvrqKwYhIqr3eGmsHvntt9+gUqmgUCjQv39/DB06FDNmzDB83r59e6OzIufPn8e1a9dga2sLlUoFlUoFJycn5OXl4fr160hJScHt27fxxBNPlGn7o0aNQmRkJFq2bIl33nkHv//+e6l9o6Ki4OfnZwhBANCjRw/o9XpcvnzZ0Na2bVtIpVLDew8PD8MlqDlz5hjqVqlUiIuLM/T74IMPEBkZiQMHDiAgIACLFi0yBIvz588jKysLDRo0MFo+NjYW169fBwBERkY+dL/PnDmDgQMHolGjRrC1tTUEhgdrKI/IyEh06tTJEIIqyt/f3+i9paUlhgwZgg0bNgAoPBP3yy+/YNiwYQAKzxjl5OTgySefNPpdfPvtt4bfBRFRfcYzQo+gtJLi0qchJtt2efTu3RsrV66ETCaDp6cnLC2ND++DoQMovBzj7+9v+JJ8kIuLS7knzuvcuTNiY2Oxe/du7N+/H0OGDEFwcHClxptYWVkZvZdIJIbLRW+++SaGDBli+MzT09Pws7OzM3x9feHr64stW7agffv26NKlC9q0aYOsrCx4eHjg0KFDxbZXdIu7Uqkstaaiy3ohISHYsGEDXFxcEBcXh5CQkAoPUH7Y9oDCSQyFEEZtJY2V+vcxBoBhw4YhKCgIKSkp2LdvH5RKJfr16wcAhjFZO3fuhJeXl9FyNTWwnYjIlBiEHkEikVTo8pQp2NjYlOtySufOnbF582a4urrCzs6uxD4+Pj6IiIhA7969y7ROOzs7DB06FEOHDsULL7yAfv364e7du8XOdLRu3Rrr1q1Ddna24cv72LFjsLCwMAzkfhQnJ6cynUHx9vbG0KFDMXXqVPzyyy/o3LkzkpKSYGlpCR8fnxKX6dChAyIiIjB69Ohin0VHR+POnTuYO3cuvL29AQCnT58uU82l6dChA9asWVPi7wooDKYXL140aouMjCwWFEvSvXt3eHt7Y/Pmzdi9ezcGDx5sWK5NmzaQy+WIi4vjZTAiMku8NGbGhg0bBmdnZzz77LM4cuQIYmNjcejQIbzzzju4desWAGDGjBn44osv8OWXX+Lq1as4e/Ysli5dWuL6Fi5ciB9++AHR0dG4cuUKtmzZAnd39xInEhw2bBgUCgVGjhyJixcv4uDBg3j77bcxfPhww/igqjRx4kT8+uuvOH36NIKDgxEYGIhBgwbh999/x40bN3D8+HF8/PHHhkATFhaGH374AWFhYYiKisKFCxcwb948AECjRo0gk8mwdOlSxMTEYMeOHZg1a1al6nvppZfg7u6OQYMG4dixY4iJicFPP/2EEydOAAD69OmD06dP49tvv8XVq1cRFhZWLBg9zMsvv4xVq1Zh3759hstiQOHdde+//z4mTZqE9evX4/r164ZjvH79+krtExFRXcAgZMasra1x+PBhNGrUCM899xxat26N1157DXl5eYYzRCNHjsTixYuxYsUKtG3bFk8//TSuXr1a4vpsbW3x+eefo0uXLujatStu3LiBXbt2lXiJzdraGnv37sXdu3fRtWtXvPDCC3jiiSewbNmyatnXNm3aoG/fvpg+fTokEgl27dqFxx9/HKNHj0aLFi3w4osv4ubNm4YQ1qtXL2zZsgU7duxAx44d0adPH5w8eRJA4dmZdevWYcuWLWjTpg3mzp2LBQsWVKo+mUyG33//Ha6urhgwYADat2+PuXPnGsZHhYSEYNq0aZg8eTK6du2KzMxMjBgxoszrHzZsGC5dugQvLy/06NHD6LNZs2Zh2rRpCA8PR+vWrdGvXz/s3LkTTZo0qdQ+ERHVBRLx74EH9ZxarYa9vT0yMjKKXQ7Ky8tDbGwsmjRpwqdqE5WCf06IyBQe9v1dGTwjRERERGaLQYiIiIjMFoMQERERmS0GISIiIjJbDEJERERkthiESmBmN9IRlQv/fBBRfcIg9ICi2XZzcnJMXAlR7VX056Mss1oTEdV2dePZETVEKpXCwcHB8FBPa2trw1PZicydEAI5OTlISUmBg4OD0cNwiYjqKgahf3F3dwcAQxgiImMODg6GPydERHUdg9C/SCQSeHh4wNXVtcSnexOZMysrK54JIqJ6pVYEoeXLl2P+/PlISkqCn58fli5dim7dupXaf8uWLZg2bRpu3LiB5s2bY968eRgwYECV1iSVSvkXPhERUT1n8sHSmzdvRmhoKMLCwnD27Fn4+fkhJCSk1EtTx48fx0svvYTXXnsN586dw6BBgzBo0KByPYmbiIiICKgFD10NCAhA165dDU8d1+v18Pb2xttvv40pU6YU6z906FBkZ2fjt99+M7T93//9Hzp27IhVq1Y9cnvV9dA2IiIiqj718qGrWq0WZ86cQXBwsKHNwsICwcHBOHHiRInLnDhxwqg/AISEhJTan4iIiKg0Jh0jlJaWBp1OBzc3N6N2Nzc3REdHl7hMUlJSif2TkpJK7K/RaKDRaAzvMzIyABQmSyIiIqobir63q/pCVq0YLF2dwsPDMXPmzGLt3t7eJqiGiIiIKuPOnTuwt7evsvWZNAg5OztDKpUiOTnZqD05ObnUeUrc3d3L1X/q1KkIDQ01vE9PT0fjxo0RFxdXpb9IKj+1Wg1vb2/Ex8dzvFYtwONRe/BY1B48FrVHRkYGGjVqBCcnpypdr0mDkEwmg7+/PyIiIjBo0CAAhYOlIyIiMGHChBKXCQwMREREBN59911D2759+xAYGFhif7lcDrlcXqzd3t6e/1PXEnZ2djwWtQiPR+3BY1F78FjUHhYWVTu82eSXxkJDQzFy5Eh06dIF3bp1w+LFi5GdnY3Ro0cDAEaMGAEvLy+Eh4cDACZOnIigoCB88cUXeOqpp7Bp0yacPn0aX3/9tSl3g4iIiOogkwehoUOHIjU1FdOnT0dSUhI6duyIPXv2GAZEx8XFGaW/7t27Y+PGjfjkk0/w0UcfoXnz5ti+fTvatWtnql0gIiKiOsrkQQgAJkyYUOqlsEOHDhVrGzx4MAYPHlyhbcnlcoSFhZV4uYxqFo9F7cLjUXvwWNQePBa1R3UdC5NPqEhERERkKiZ/xAYRERGRqTAIERERkdliECIiIiKzxSBEREREZqteBqHly5fDx8cHCoUCAQEBOHny5EP7b9myBa1atYJCoUD79u2xa9euGqq0/ivPsVi9ejV69uwJR0dHODo6Ijg4+JHHjsqnvH82imzatAkSicQw8SlVXnmPRXp6OsaPHw8PDw/I5XK0aNGCf1dVkfIei8WLF6Nly5ZQKpXw9vbGpEmTkJeXV0PV1l+HDx/GwIED4enpCYlEgu3btz9ymUOHDqFz586Qy+Xw9fXFunXryr9hUc9s2rRJyGQy8c0334i///5bvP7668LBwUEkJyeX2P/YsWNCKpWKzz//XFy6dEl88sknwsrKSly4cKGGK69/ynssXn75ZbF8+XJx7tw5ERUVJUaNGiXs7e3FrVu3arjy+qm8x6NIbGys8PLyEj179hTPPvtszRRbz5X3WGg0GtGlSxcxYMAAcfToUREbGysOHTokIiMja7jy+qe8x2LDhg1CLpeLDRs2iNjYWLF3717h4eEhJk2aVMOV1z+7du0SH3/8sfj5558FALFt27aH9o+JiRHW1tYiNDRUXLp0SSxdulRIpVKxZ8+ecm233gWhbt26ifHjxxve63Q64enpKcLDw0vsP2TIEPHUU08ZtQUEBIixY8dWa53moLzH4t8KCgqEra2tWL9+fXWVaFYqcjwKCgpE9+7dxZo1a8TIkSMZhKpIeY/FypUrRdOmTYVWq62pEs1GeY/F+PHjRZ8+fYzaQkNDRY8ePaq1TnNTliA0efJk0bZtW6O2oUOHipCQkHJtq15dGtNqtThz5gyCg4MNbRYWFggODsaJEydKXObEiRNG/QEgJCSk1P5UNhU5Fv+Wk5OD/Pz8Kn/Anjmq6PH49NNP4erqitdee60myjQLFTkWO3bsQGBgIMaPHw83Nze0a9cOc+bMgU6nq6my66WKHIvu3bvjzJkzhstnMTEx2LVrFwYMGFAjNdM/qur7u1bMLF1V0tLSoNPpDI/nKOLm5obo6OgSl0lKSiqxf1JSUrXVaQ4qciz+7cMPP4Snp2ex/9Gp/CpyPI4ePYr//ve/iIyMrIEKzUdFjkVMTAwOHDiAYcOGYdeuXbh27RrGjRuH/Px8hIWF1UTZ9VJFjsXLL7+MtLQ0PPbYYxBCoKCgAG+++SY++uijmiiZHlDa97darUZubi6USmWZ1lOvzghR/TF37lxs2rQJ27Ztg0KhMHU5ZiczMxPDhw/H6tWr4ezsbOpyzJ5er4erqyu+/vpr+Pv7Y+jQofj444+xatUqU5dmdg4dOoQ5c+ZgxYoVOHv2LH7++Wfs3LkTs2bNMnVpVEH16oyQs7MzpFIpkpOTjdqTk5Ph7u5e4jLu7u7l6k9lU5FjUWTBggWYO3cu9u/fjw4dOlRnmWajvMfj+vXruHHjBgYOHGho0+v1AABLS0tcvnwZzZo1q96i66mK/Nnw8PCAlZUVpFKpoa1169ZISkqCVquFTCar1prrq4oci2nTpmH48OEYM2YMAKB9+/bIzs7GG2+8gY8//tjoIeFUvUr7/razsyvz2SCgnp0Rkslk8Pf3R0REhKFNr9cjIiICgYGBJS4TGBho1B8A9u3bV2p/KpuKHAsA+PzzzzFr1izs2bMHXbp0qYlSzUJ5j0erVq1w4cIFREZGGl7PPPMMevfujcjISHh7e9dk+fVKRf5s9OjRA9euXTOEUQC4cuUKPDw8GIIqoSLHIicnp1jYKQqogo/urFFV9v1dvnHctd+mTZuEXC4X69atE5cuXRJvvPGGcHBwEElJSUIIIYYPHy6mTJli6H/s2DFhaWkpFixYIKKiokRYWBhvn68i5T0Wc+fOFTKZTGzdulUkJiYaXpmZmabahXqlvMfj33jXWNUp77GIi4sTtra2YsKECeLy5cvit99+E66uruKzzz4z1S7UG+U9FmFhYcLW1lb88MMPIiYmRvz++++iWbNmYsiQIabahXojMzNTnDt3Tpw7d04AEAsXLhTnzp0TN2/eFEIIMWXKFDF8+HBD/6Lb5z/44AMRFRUlli9fztvniyxdulQ0atRIyGQy0a1bN/Hnn38aPgsKChIjR4406v/jjz+KFi1aCJlMJtq2bSt27txZwxXXX+U5Fo0bNxYAir3CwsJqvvB6qrx/Nh7EIFS1ynssjh8/LgICAoRcLhdNmzYVs2fPFgUFBTVcdf1UnmORn58vZsyYIZo1ayYUCoXw9vYW48aNE/fu3av5wuuZgwcPlvgdUPT7HzlypAgKCiq2TMeOHYVMJhNNmzYVa9euLfd2JULwXB4RERGZp3o1RoiIiIioPBiEiIiIyGwxCBEREZHZYhAiIiIis8UgRERERGaLQYiIiIjMFoMQERERmS0GISIiABKJBNu3bwcA3LhxAxKJBJGRkSatiYiqH4MQEZncqFGjIJFIIJFIYGVlhSZNmmDy5MnIy8szdWlEVM/Vq6fPE1Hd1a9fP6xduxb5+fk4c+YMRo4cCYlEgnnz5pm6NCKqx3hGiIhqBblcDnd3d3h7e2PQoEEIDg7Gvn37ABQ+ETw8PBxNmjSBUqmEn58ftm7darT833//jaeffhp2dnawtbVFz549cf36dQDAqVOn8OSTT8LZ2Rn29vYICgrC2bNna3wfiaj2YRAiolrn4sWLOH78OGQyGQAgPDwc3377LVatWoW///4bkyZNwiuvvII//vgDAJCQkIDHH38ccrkcBw4cwJkzZ/Dqq6+ioKAAAJCZmYmRI0fi6NGj+PPPP9G8eXMMGDAAmZmZJttHIqodeGmMiGqF3377DSqVCgUFBdBoNLCwsMCyZcug0WgwZ84c7N+/H4GBgQCApk2b4ujRo/jqq68QFBSE5cuXw97eHps2bYKVlRUAoEWLFoZ19+nTx2hbX3/9NRwcHPDHH3/g6aefrrmdJKJah0GIiGqF3r17Y+XKlcjOzsaiRYtgaWmJ559/Hn///TdycnLw5JNPGvXXarXo1KkTACAyMhI9e/Y0hKB/S05OxieffIJDhw4hJSUFOp0OOTk5iIuLq/b9IqLajUGIiGoFGxsb+Pr6AgC++eYb+Pn54b///S/atWsHANi5cye8vLyMlpHL5QAApVL50HWPHDkSd+7cwZIlS9C4cWPI5XIEBgZCq9VWw54QUV3CIEREtY6FhQU++ugjhIaG4sqVK5DL5YiLi0NQUFCJ/Tt06ID169cjPz+/xLNCx44dw4oVKzBgwAAAQHx8PNLS0qp1H4iobuBgaSKqlQYPHgypVIqvvvoK77//PiZNmoT169fj+vXrOHv2LJYuXYr169cDACZMmAC1Wo0XX3wRp0+fxtWrV/Hdd9/h8uXLAIDmzZvju+++Q1RUFP766y8MGzbskWeRiMg88IwQEdVKlpaWmDBhAj7//HPExsbCxcUF4eHhiImJgYODAzp37oyPPvoIANCgQQMcOHAAH3zwAYKCgiCVStGxY0f06NEDAPDf//4Xb7zxBjp37gxvb2/MmTMH77//vil3j4hqCYkQQpi6CCIiIiJT4KUxIiIiMlsMQkRERGS2GISIiIjIbDEIERERkdliECIiIiKzxSBEREREZotBiIiIiMwWgxARERGZLQYhIiIiMlsMQkRERGS2GISIiIjIbDEIERERkdn6fwShlKxyisqeAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# Compute Precision-Recall and plot curve, based on test_predictions and test_labels\n",
    "y_test = np.array(test_labels)\n",
    "y_score = np.array(test_predictions)\n",
    "\n",
    "precision, recall, _ = precision_recall_curve(y_test, y_score)\n",
    "average_precision = average_precision_score(y_test, y_score)\n",
    "\n",
    "# Plot\n",
    "plt.clf()\n",
    "plt.plot(recall, precision, label='Precision-Recall curve')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('Precision-Recall, AUC={0:0.2f}'.format(average_precision))\n",
    "plt.legend(loc=\"lower left\")\n",
    "\n",
    "# Save the plot\n",
    "fig_path = f\"{session_label}_{output_epoch}_prec-recall.png\"\n",
    "plt.savefig(fig_path)\n",
    "\n",
    "# AFTER saving\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.metrics import roc_curve, auc\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.preprocessing import label_binarize\n",
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "# # # Assuming you have your features in X and target in y\n",
    "# # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# # Binarize the output\n",
    "# y_test = label_binarize(y_test, classes=[0, 1])\n",
    "\n",
    "# # Train your classifier and predict probabilities\n",
    "# # clf.fit(X_train, y_train)\n",
    "# y_score = clf.predict_proba(X_test)\n",
    "\n",
    "# # Compute ROC curve and ROC area for each class\n",
    "# fpr, tpr, _ = roc_curve(y_test, y_score[:, 1])\n",
    "# roc_auc = auc(fpr, tpr)\n",
    "\n",
    "# # Plot\n",
    "# plt.figure()\n",
    "# lw = 2\n",
    "# plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "# plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "# plt.xlim([0.0, 1.0])\n",
    "# plt.ylim([0.0, 1.05])\n",
    "# plt.xlabel('False Positive Rate')\n",
    "# plt.ylabel('True Positive Rate')\n",
    "# plt.title('Receiver Operating Characteristic example')\n",
    "# plt.legend(loc=\"lower right\")\n",
    "# plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "qu",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}