value_aware_AL.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "value_aware_AL.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "phPjYIkiljiY"
      },
      "outputs": [],
      "source": [
        "!pip install sklearn\n",
        "!pip install sentence-transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Mount Drive into Colab\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "qwIM54CgmNUD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Import libraries\n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "import csv\n",
        "import random\n",
        "from random import shuffle\n",
        "from scipy.special import softmax\n",
        "from sklearn.metrics import precision_recall_fscore_support, accuracy_score\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer, CountVectorizer"
      ],
      "metadata": {
        "id": "ZM60D1XsmOEa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "random.seed(41)\n",
        "# define parameters\n",
        "datasetName = 'usAirline'                                                           # name of the dataset\n",
        "data_folder = '.../usAirline/data/'     # define the path where you store your datasets\n",
        "dataToTrain = 'usAirline_train.csv'                                                 # name of your training data file\n",
        "dataToVal = 'usAirline_val.csv'                                                     # name of your validation data file\n",
        "dataToTest ='usAirline_test.csv'                                                    # name of your test data file\n",
        "txt = 'text'                                                                        # define the column name of your text data\n",
        "goldLabel = 'airline_sentiment'                                                     # define the column name of your ground truth label\n",
        "\n",
        "# AL parameters\n",
        "al_strategies           = ['uncertainty', 'certainty', 'random', 'tos', 'tos-below'] # define active learning strategies you want to test\n",
        "minimum_training_items  = 3                                                          # minimum number of training items before we first train a model\n",
        "alBatchNum              = 88                                                         # define the total number of batches in active learning pipeline\n",
        "alBatchSize             = 100                                                        # define the size of one batch in active learning pipeline\n",
        "controlList2 = [25, 50, 88]                                                          # define on which batches you want to save the predicted probabilities as a separate file \n",
        "\n",
        "\n",
        "encName = 'tfidf'                                                                    # pick either 'tfidf' or 'mpnet' as the encoder\n",
        "model = LogisticRegression(max_iter=1000)                                            # we use Logistic Regression model, you can modify this part\n",
        "modelName = 'LogReg'                                                                 # name of your model\n",
        "\n",
        "\n",
        "iID = 'itemID'                                                                              # do not change this column\n",
        "res_path = '.../AL/res/{}/'.format(datasetName) # specify the path to keep results\n",
        "logfile_name = \"{}_{}_rnd41_{}_\".format(datasetName,modelName,encName)       "
      ],
      "metadata": {
        "id": "NjeId6BSmdRp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#cost-based parameters\n",
        "Vr = 0.0\n",
        "Vc = 1.0\n",
        "Vw_list = [0, -0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -2.0, -4.0, -8.0, -10.0, -100.0]\n",
        "confT_list = list(np.arange(0, 1.01, 0.01))"
      ],
      "metadata": {
        "id": "R6jS7a4ooRKT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create the encoder\n",
        "if encName == 'tfidf':\n",
        "    encoder = TfidfVectorizer(min_df=0, max_features = 1024, strip_accents='unicode', analyzer='word', token_pattern=r'\\w{1,}',\n",
        "                ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,\n",
        "                stop_words='english', lowercase=False)\n",
        "elif encName == 'mpnet':\n",
        "    encoder = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')\n",
        "else:\n",
        "    raise Exception(\"The encoder name provided can't be recognised. You should pick either tfidf or mpnet as the text encoder\")"
      ],
      "metadata": {
        "id": "BMnP1qfJo-i0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CELoss(object):\n",
        "\n",
        "    def compute_bin_boundaries(self, probabilities = np.array([])):\n",
        "\n",
        "        #uniform bin spacing\n",
        "        if probabilities.size == 0:\n",
        "            bin_boundaries = np.linspace(0, 1, self.n_bins + 1)\n",
        "            self.bin_lowers = bin_boundaries[:-1]\n",
        "            self.bin_uppers = bin_boundaries[1:]\n",
        "        else:\n",
        "            #size of bins \n",
        "            bin_n = int(self.n_data/self.n_bins)\n",
        "\n",
        "            bin_boundaries = np.array([])\n",
        "\n",
        "            probabilities_sort = np.sort(probabilities)  \n",
        "\n",
        "            for i in range(0,self.n_bins):\n",
        "                bin_boundaries = np.append(bin_boundaries,probabilities_sort[i*bin_n])\n",
        "            bin_boundaries = np.append(bin_boundaries,1.0)\n",
        "\n",
        "            self.bin_lowers = bin_boundaries[:-1]\n",
        "            self.bin_uppers = bin_boundaries[1:]\n",
        "\n",
        "\n",
        "    def get_probabilities(self, output, labels, logits):\n",
        "        #If not probabilities apply softmax!\n",
        "        if logits:\n",
        "            self.probabilities = softmax(output, axis=1)\n",
        "        else:\n",
        "            self.probabilities = output\n",
        "\n",
        "        self.labels = labels\n",
        "        self.confidences = np.max(self.probabilities, axis=1)\n",
        "        self.predictions = np.argmax(self.probabilities, axis=1)\n",
        "        self.accuracies = np.equal(self.predictions,labels)\n",
        "\n",
        "    def binary_matrices(self):\n",
        "        idx = np.arange(self.n_data)\n",
        "        #make matrices of zeros\n",
        "        pred_matrix = np.zeros([self.n_data,self.n_class])\n",
        "        label_matrix = np.zeros([self.n_data,self.n_class])\n",
        "        #self.acc_matrix = np.zeros([self.n_data,self.n_class])\n",
        "        pred_matrix[idx,self.predictions] = 1\n",
        "        label_matrix[idx,self.labels] = 1\n",
        "\n",
        "        self.acc_matrix = np.equal(pred_matrix, label_matrix)\n",
        "\n",
        "\n",
        "    def compute_bins(self, index = None):\n",
        "        self.bin_prop = np.zeros(self.n_bins)\n",
        "        self.bin_acc = np.zeros(self.n_bins)\n",
        "        self.bin_conf = np.zeros(self.n_bins)\n",
        "        self.bin_score = np.zeros(self.n_bins)\n",
        "\n",
        "        if index == None:\n",
        "            confidences = self.confidences\n",
        "            accuracies = self.accuracies\n",
        "        else:\n",
        "            confidences = self.probabilities[:,index]\n",
        "            accuracies = self.acc_matrix[:,index]\n",
        "\n",
        "\n",
        "        for i, (bin_lower, bin_upper) in enumerate(zip(self.bin_lowers, self.bin_uppers)):\n",
        "            # Calculated |confidence - accuracy| in each bin\n",
        "            in_bin = np.greater(confidences,bin_lower.item()) * np.less_equal(confidences,bin_upper.item())\n",
        "            self.bin_prop[i] = np.mean(in_bin)\n",
        "\n",
        "            if self.bin_prop[i].item() > 0:\n",
        "                self.bin_acc[i] = np.mean(accuracies[in_bin])\n",
        "                self.bin_conf[i] = np.mean(confidences[in_bin])\n",
        "                self.bin_score[i] = np.abs(self.bin_conf[i] - self.bin_acc[i])\n",
        "\n",
        "class MaxProbCELoss(CELoss):\n",
        "    def loss(self, output, labels, n_bins = 15, logits = True):\n",
        "        self.n_bins = n_bins\n",
        "        super().compute_bin_boundaries()\n",
        "        super().get_probabilities(output, labels, logits)\n",
        "        super().compute_bins()\n",
        "\n",
        "#http://people.cs.pitt.edu/~milos/research/AAAI_Calibration.pdf\n",
        "class ECELoss(MaxProbCELoss):\n",
        "\n",
        "    def loss(self, output, labels, n_bins = 15, logits = False):\n",
        "        super().loss(output, labels, n_bins, logits)\n",
        "        return np.dot(self.bin_prop,self.bin_score)"
      ],
      "metadata": {
        "id": "_9osH6ILrEvZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def uncertainty_sampling(model, unl_emb, number):\n",
        "    \"\"\"Returns batch of datapoints with smallest margin/highest uncertainty.\n",
        "    For binary classification, can just take the absolute distance to decision\n",
        "    boundary for each point.\n",
        "    For multiclass classification, must consider the margin between distance for\n",
        "    top two most likely classes.\n",
        "    Returns:\n",
        "      indices of points selected to add using margin active learner\n",
        "    \"\"\"\n",
        "    '''Points are sampled according to uncertainty sampling criterion'''\n",
        "\n",
        "    distances = model.predict_proba(unl_emb)\n",
        "\n",
        "    if len(distances.shape) < 2:\n",
        "      min_margin = abs(distances)\n",
        "    else:\n",
        "      sort_distances = np.sort(distances, 1)[:, -2:]\n",
        "      min_margin = sort_distances[:, 1] - sort_distances[:, 0]\n",
        "    score_indices = np.argsort(min_margin)\n",
        "    selected_samples = score_indices[0:number]\n",
        "    return selected_samples\n",
        "\n",
        "def certainty_sampling(model, unl_emb, number):\n",
        "    \"\"\"Returns batch of datapoints with highest margin/smallest uncertainty.\n",
        "    For binary classification, can just take the absolute distance to decision\n",
        "    boundary for each point.\n",
        "    For multiclass classification, must consider the margin between distance for\n",
        "    top two most likely classes.\n",
        "    Returns:\n",
        "      indices of points selected to add using margin active learner\n",
        "    \"\"\"\n",
        "    '''Points are sampled according to certainty sampling criterion'''\n",
        "\n",
        "    distances = model.predict_proba(unl_emb)\n",
        "\n",
        "    if len(distances.shape) < 2:\n",
        "      min_margin = abs(distances)\n",
        "    else:\n",
        "      sort_distances = np.sort(distances, 1)[:, -2:]\n",
        "      min_margin = sort_distances[:, 1] - sort_distances[:, 0]\n",
        "    score_indices = np.argsort(min_margin)\n",
        "    score_indices_reversed = score_indices[::-1]\n",
        "    selected_samples = score_indices_reversed[0:number]\n",
        "    return selected_samples\n",
        "\n",
        "def random_sampling(dataIds, nQuery):\n",
        "    '''Randomly samples the points'''\n",
        "    query_idx = random.sample(range(len(dataIds)), nQuery)\n",
        "    selectedIndex = dataIds[query_idx]\n",
        "    return selectedIndex\n",
        "\n",
        "def threshold_oriented_sampling(model, unl_emb, number, t):\n",
        "    probs = model.predict_proba(unl_emb)\n",
        "    margins = np.array([abs(np.amax(l) - t) for l in probs])\n",
        "    score_indices = np.argsort(margins)\n",
        "    selected_samples = score_indices[0:number]\n",
        "    return selected_samples\n",
        "\n",
        "def tos_below(model, unl_emb, number, t):\n",
        "    probs = model.predict_proba(unl_emb)\n",
        "    indices_acc = np.array([1 if np.amax(l) > t else 0 for l in probs])\n",
        "    indices_accepted = np.array(np.where(indices_acc == 1)[0])\n",
        "    margins = np.array([abs(np.amax(l) - t) for l in probs])\n",
        "    score_indices = np.argsort(margins)\n",
        "    score_indices = np.setdiff1d(score_indices, indices_accepted)\n",
        "    selected_samples = score_indices[0:number]\n",
        "    return selected_samples\n",
        "\n",
        "def prepare_features(tfidf, X_train, setN):\n",
        "    # compute tfidf features\n",
        "    if setN == 'train':\n",
        "        X_train_tfidf = tfidf.fit_transform(X_train).toarray()\n",
        "    else:\n",
        "        X_train_tfidf = tfidf.transform(X_train).toarray()\n",
        "    return X_train_tfidf\n",
        "\n",
        "class Data():\n",
        "    \n",
        "    def __init__(self, encoder, filename, setN, encoderName):\n",
        "        \n",
        "        # each dataset will have a pool of data, together with their IDs and gold labels \n",
        "        self.poolData = np.array([])\n",
        "        self.poolGoldLabels = np.array([])\n",
        "        \n",
        "        dt = pd.read_csv(filename)\n",
        "        #dt = dt.dropna()     # uncomment this if your data has none values\n",
        "        dt = dt.reset_index(drop=True)\n",
        "        dt['itemID'] = np.arange(dt.shape[0])\n",
        "        y = dt[goldLabel].values\n",
        "        if encName == 'tfidf':\n",
        "            X = prepare_features(encoder, dt[txt].tolist(), setN)\n",
        "        else:\n",
        "            X = encoder.encode(dt[txt].tolist())\n",
        "        self.data = dt\n",
        "        self.poolDataEmb = X\n",
        "        self.poolGoldLabels = y\n",
        "        self.mClass = list(set(self.poolGoldLabels.tolist()))\n",
        "        \n",
        "    def setStartState(self, nStart):\n",
        "        ''' This functions creates the initial training set which contains the equal number of samples per class\n",
        "        Input:\n",
        "        nStart -- number of labelled datapoints (size of training set)\n",
        "        '''\n",
        "        self.nStart = nStart\n",
        "        data = self.data.copy()\n",
        "        # get predefined points so that all classes are represented and initial classifier could be trained.\n",
        "        sampledIndices = []\n",
        "        for cls in self.mClass:\n",
        "            indices = np.array(np.where(self.poolGoldLabels == cls)).tolist()[0]\n",
        "            idx = random.sample(indices, nStart // len(mClass))\n",
        "            sampledIndices = sampledIndices + idx\n",
        "\n",
        "        sData = data.iloc[sampledIndices]\n",
        "        self.labeledSet = sData.reset_index(drop=True)\n",
        "        droppedData = data.drop(sampledIndices)\n",
        "        self.unlabeledSet = droppedData.reset_index(drop=True)"
      ],
      "metadata": {
        "id": "tyKcel23p1se"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_model(model, X_train, y_train, X_val, y_val, X_test, y_test, rp_path, batch):\n",
        "\n",
        "    y_pred_train = model.predict(X_train)\n",
        "    logits_train = model.predict_proba(X_train)\n",
        "\n",
        "    y_pred_val = model.predict(X_val)\n",
        "    logits_val = model.predict_proba(X_val)\n",
        "\n",
        "    y_pred_test = model.predict(X_test)\n",
        "    logits_test = model.predict_proba(X_test)\n",
        "    \n",
        "    # check if binary or multi class classification\n",
        "    num_classes = len(set(y_test))\n",
        "    if num_classes == 2:\n",
        "        average = 'binary'\n",
        "    else:\n",
        "        average = 'macro'\n",
        " \n",
        "    acc_train = accuracy_score(y_train, y_pred_train)\n",
        "    acc_val = accuracy_score(y_val, y_pred_val)\n",
        "    acc_test = accuracy_score(y_test, y_pred_test)\n",
        "    pre_train, rec_train, f1_train, _ = precision_recall_fscore_support(y_train, y_pred_train, average=average, beta=1)\n",
        "    pre_val, rec_val, f1_val, _ = precision_recall_fscore_support(y_val, y_pred_val, average=average, beta=1)\n",
        "    pre_test, rec_test, f1_test, _ = precision_recall_fscore_support(y_test, y_pred_test, average=average, beta=1)\n",
        "    \n",
        "    ece = ECELoss()\n",
        "    ece_train = ece.loss(logits_train, y_train, logits=False)\n",
        "    ece_val = ece.loss(logits_val, y_val, logits=False)\n",
        "    ece_test = ece.loss(logits_test, y_test, logits=False)\n",
        "\n",
        "    with open(rp_path, 'a') as f:\n",
        "        res_i = '{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\\n'.format(batch, acc_train, pre_train, rec_train, f1_train, ece_train, acc_val, pre_val, rec_val, f1_val, ece_val, acc_test, pre_test, rec_test, f1_test, ece_test)\n",
        "        f.write(res_i)\n",
        "\n",
        "    return logits_train, logits_val, logits_test\n",
        "\n",
        "def cost_based_threshold(k):\n",
        "    t = (k)/(k+1)\n",
        "    return t\n",
        "\n",
        "def calculate_value(y_hat_proba, y, t, Vr, Vc, Vw):\n",
        "\n",
        "    y_pred = np.array([np.where(l == np.amax(l))[0][0] if (np.amax(l) > t) else -1 for l in y_hat_proba])\n",
        "\n",
        "    # now lets compute the actual value of each prediction\n",
        "    \n",
        "    value_vector = np.full(y_pred.shape[0], Vc)\n",
        "\n",
        "    value_vector[(y_pred != y) & (y_pred != -1)] = Vw\n",
        "    \n",
        "    #loss due to asking humans\n",
        "    value_vector[y_pred == -1] = Vr\n",
        "    value = np.sum(value_vector) / len(y)\n",
        "\n",
        "    numOfRejectedSamples = np.count_nonzero(y_pred == -1)\n",
        "    numOfWrongPredictions = np.count_nonzero((y_pred != y) & (y_pred != -1))\n",
        "    return value, numOfRejectedSamples, numOfWrongPredictions\n",
        "\n",
        "def find_optimum_confidence_threshold(y_hat_proba, y, t_list, Vr, Vc, Vw):\n",
        "\n",
        "    cost_list = {}\n",
        "\n",
        "    for t in t_list:\n",
        "        value, _ , __ = calculate_value(y_hat_proba, y, t, Vr, Vc, Vw)\n",
        "        cost_list[\"{}\".format(t)] = value\n",
        "    # find t values with maximum value\n",
        "    maxValue = max(cost_list.values())\n",
        "    optTList = [float(k) for k, v in cost_list.items() if v == maxValue]\n",
        "    # pick the one with the lowest confidence\n",
        "    optimumT = min(optTList)\n",
        "\n",
        "    return optimumT, cost_list"
      ],
      "metadata": {
        "id": "OUACADRKqit1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def run_sota_AL(model,al_strategy,pool,mClass,validation,validation_data,y_val,test,test_data,y_test,alBatchSize,res_path,logfile_name,Vw_list):\n",
        "    poolData = pool.data.copy()\n",
        "    training_data = pool.labeledSet.copy()\n",
        "    unlabeled_data = pool.unlabeledSet.copy()\n",
        "    batchSize = alBatchSize\n",
        "\n",
        "    poolDataEmb_val = validation.poolDataEmb\n",
        "    poolDataEmb_test = test.poolDataEmb\n",
        "\n",
        "    train_data = pool.poolDataEmb[poolData.index[poolData[iID].isin(training_data[iID].values)].tolist()]\n",
        "    train_labels = np.array(training_data[goldLabel].tolist())\n",
        "\n",
        "    #Start active learning\n",
        "    sampleIds = []\n",
        "    samplingRanks = []\n",
        "    samplesDict = {}\n",
        "    samplesDict[0] = training_data[iID].tolist()\n",
        "\n",
        "    # create log file \n",
        "    rp_path = res_path + logfile_name + al_strategy + \"_perf.csv\"\n",
        "    with open(rp_path, 'w') as f:\n",
        "        c = 'batch, acc_train, pre_train, rec_train, f1_train, ece_train, acc_val, pre_val, rec_val, f1_val, ece_val, acc_test, pre_test, rec_test, f1_test, ece_test'\n",
        "        f.write(c + '\\n')\n",
        "\n",
        "    rv_path = res_path + logfile_name + al_strategy + \"_value.csv\"\n",
        "    with open(rv_path, 'w') as f:\n",
        "        c = 'batch, Vr, Vc, Vw, k, t_cal, t_opt_val, t_opt_train, t_opt_test, value_test, rej_test, wrong_test, value_train, rej_train, wrong_train, value_test_opt, rej_test_opt, wrong_test_opt, value_train_opt, rej_train_opt, wrong_train_opt, value_test_opt_test, rej_test_opt_test, wrong_test_opt_test'\n",
        "        f.write(c + '\\n')\n",
        "\n",
        "    model.fit(train_data, train_labels) \n",
        "    logits_train, logits_val, logits_test = evaluate_model(model, train_data, train_labels, poolDataEmb_val, y_val, poolDataEmb_test, y_test, rp_path, 0)\n",
        "\n",
        "    for Vw in Vw_list:\n",
        "        k = (-1)*(Vw / Vc)\n",
        "        t = cost_based_threshold(k)\n",
        "        value_test, rej_test, wrong_test = calculate_value(logits_test, y_test, t, Vr, Vc, Vw)\n",
        "        value_train, rej_train, wrong_train = calculate_value(logits_train, train_labels, t, Vr, Vc, Vw)\n",
        "\n",
        "        t_opt, cost_list = find_optimum_confidence_threshold(logits_val, y_val, confT_list, Vr, Vc, Vw)\n",
        "        value_test_opt, rej_test_opt, wrong_test_opt = calculate_value(logits_test, y_test, t_opt, Vr, Vc, Vw)\n",
        "        value_train_opt, rej_train_opt, wrong_train_opt  = calculate_value(logits_train, train_labels, t_opt, Vr, Vc, Vw)\n",
        "\n",
        "        t_opt_train, cost_list_ = find_optimum_confidence_threshold(logits_train, train_labels, confT_list, Vr, Vc, Vw)\n",
        "        t_opt_test, cost_list_ = find_optimum_confidence_threshold(logits_test, y_test, confT_list, Vr, Vc, Vw)\n",
        "        value_test_opt_test, rej_test_opt_test, wrong_test_opt_test = calculate_value(logits_test, y_test, t_opt_test, Vr, Vc, Vw)\n",
        "\n",
        "        with open(rv_path, 'a') as f:\n",
        "            res_i = '{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\\n'.format(0, Vr, Vc, Vw, k, t, t_opt, t_opt_train, t_opt_test, value_test, rej_test, wrong_test, value_train, rej_train, wrong_train, value_test_opt, rej_test_opt, wrong_test_opt, value_train_opt, rej_train_opt, wrong_train_opt, value_test_opt_test, rej_test_opt_test, wrong_test_opt_test)\n",
        "            f.write(res_i)\n",
        "    \n",
        "    for alBatch in range(1, alBatchNum + 1, 1):\n",
        "        sampledIndices = []\n",
        "\n",
        "        unl_dataEmb = pool.poolDataEmb[poolData.index[poolData[iID].isin(unlabeled_data[iID].values)].tolist()]\n",
        "\n",
        "        if alBatch == alBatchNum:\n",
        "            batchSize = len(unlabeled_data[iID].values)\n",
        "            print(\"alBatchSize changed to: \", batchSize)\n",
        "\n",
        "        if al_strategy == 'uncertainty':\n",
        "            idx = uncertainty_sampling(model, unl_dataEmb, batchSize)\n",
        "            sampledIndices = unlabeled_data.loc[idx][iID].tolist()\n",
        "            for i in sampledIndices: sampleIds.append(i)\n",
        "        elif al_strategy == 'random':\n",
        "            sampledIndices = random_sampling(unlabeled_data[iID].values, batchSize)\n",
        "            for i in sampledIndices: sampleIds.append(i)\n",
        "        else:\n",
        "            #default sampling, random\n",
        "            sampledIndices = random_sampling(unlabeled_data[iID].values, batchSize)\n",
        "            for i in sampledIndices: sampleIds.append(i)\n",
        "\n",
        "        sampledSet = poolData.loc[poolData[iID].isin(sampledIndices)]\n",
        "        samplesDict[alBatch] = sampledIndices\n",
        "                    \n",
        "        training_data.reset_index(drop=True)\n",
        "        sampledSet.reset_index(drop=True)\n",
        "        training_data = pd.concat([training_data, sampledSet], axis=0).reset_index(drop=True)\n",
        "        training_data = training_data.sort_values(iID)\n",
        "        indices = unlabeled_data.loc[unlabeled_data[iID].isin(sampledIndices)].index.to_list()\n",
        "        unlabeled_data = unlabeled_data.drop(indices).reset_index(drop=True)\n",
        "        unlabeled_data = unlabeled_data.reset_index(drop=True)\n",
        "  \n",
        "        train_data = pool.poolDataEmb[poolData.index[poolData[iID].isin(training_data[iID].values)].tolist()]\n",
        "        train_labels = np.array(training_data[goldLabel].tolist())\n",
        "\n",
        "        model.fit(train_data, train_labels) \n",
        "\n",
        "        logits_train, logits_val, logits_test = evaluate_model(model, train_data, train_labels, poolDataEmb_val, y_val, poolDataEmb_test, y_test, rp_path, alBatch)\n",
        "\n",
        "        if alBatch in controlList2:\n",
        "            col = []\n",
        "            for i in range(logits_test.shape[1]):\n",
        "                col.append(str(i))\n",
        "            df_lgt = pd.DataFrame(logits_test, columns = col)\n",
        "            df_lgt['y'] = y_test\n",
        "            df_lgt.to_csv(res_path + logfile_name + al_strategy + '_b_{}_logits.csv'.format(alBatch))\n",
        "\n",
        "        for Vw in Vw_list:\n",
        "            k = (-1)*(Vw / Vc)\n",
        "            t = cost_based_threshold(k)\n",
        "\n",
        "            value_test, rej_test, wrong_test = calculate_value(logits_test, y_test, t, Vr, Vc, Vw)\n",
        "            value_train, rej_train, wrong_train = calculate_value(logits_train, train_labels,  t, Vr, Vc, Vw)\n",
        "\n",
        "            t_opt, cost_list = find_optimum_confidence_threshold(logits_val, y_val, confT_list, Vr, Vc, Vw)\n",
        "\n",
        "            value_test_opt, rej_test_opt, wrong_test_opt = calculate_value(logits_test, y_test, t_opt, Vr, Vc, Vw)\n",
        "            value_train_opt, rej_train_opt, wrong_train_opt  = calculate_value(logits_train, train_labels, t_opt, Vr, Vc, Vw)\n",
        "\n",
        "            t_opt_train, cost_list_ = find_optimum_confidence_threshold(logits_train, train_labels, confT_list, Vr, Vc, Vw)\n",
        "            t_opt_test, cost_list_ = find_optimum_confidence_threshold(logits_test, y_test, confT_list, Vr, Vc, Vw)\n",
        "            value_test_opt_test, rej_test_opt_test, wrong_test_opt_test = calculate_value(logits_test, y_test, t_opt_test, Vr, Vc, Vw)\n",
        "\n",
        "            with open(rv_path, 'a') as f:\n",
        "                res_i = '{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\\n'.format(alBatch, Vr, Vc, Vw, k, t, t_opt, t_opt_train, t_opt_test, value_test, rej_test, wrong_test, value_train, rej_train, wrong_train, value_test_opt, rej_test_opt, wrong_test_opt, value_train_opt, rej_train_opt, wrong_train_opt, value_test_opt_test, rej_test_opt_test, wrong_test_opt_test)\n",
        "                f.write(res_i) \n",
        "        \n",
        "  #  training_data.to_csv(res_path + logfile_name + al_strategy + \"_trainingData.csv\")\n",
        "    samplesDict_df = pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in samplesDict.items() ]))\n",
        "    samplesDict_df.to_csv(res_path + logfile_name + al_strategy + \"_sampledItems.csv\")\n",
        "\n",
        "\n",
        "def run_tos(model,al_strategy,pool,mClass,validation,validation_data,y_val,test,test_data,y_test,alBatchSize,res_path,logfile_name,Vw_list):\n",
        "    poolData = pool.data.copy()\n",
        "    training_data_first = pool.labeledSet\n",
        "    unlabeled_data_first = pool.unlabeledSet\n",
        "    batchSize = alBatchSize\n",
        "\n",
        "    poolDataEmb_val = validation.poolDataEmb\n",
        "    poolDataEmb_test = test.poolDataEmb\n",
        "\n",
        "    train_data = pool.poolDataEmb[poolData.index[poolData[iID].isin(training_data_first[iID].values)].tolist()]\n",
        "    train_labels = np.array(training_data_first[goldLabel].tolist())\n",
        "\n",
        "    #Start active learning\n",
        "    samplingRanks = []\n",
        "    samplesDict = {}\n",
        "    samplesDict[0] = training_data_first[iID].tolist()\n",
        "\n",
        "    model.fit(train_data, train_labels) \n",
        "\n",
        "    for Vw in Vw_list:\n",
        "        k = (-1)*(Vw / Vc)\n",
        "        t = cost_based_threshold(k)\n",
        "\n",
        "        rp_path = res_path + logfile_name + al_strategy + '_' + str(k) + \"_perf.csv\"\n",
        "        with open(rp_path, 'w') as f:\n",
        "            c = 'batch, acc_train, pre_train, rec_train, f1_train, ece_train, acc_val, pre_val, rec_val, f1_val, ece_val, acc_test, pre_test, rec_test, f1_test, ece_test'\n",
        "            f.write(c + '\\n')\n",
        "\n",
        "        logits_train, logits_val, logits_test = evaluate_model(model, train_data, train_labels, poolDataEmb_val, y_val, poolDataEmb_test, y_test, rp_path, 0)\n",
        "\n",
        "\n",
        "        value_test, rej_test, wrong_test = calculate_value(logits_test, y_test, t, Vr, Vc, Vw)\n",
        "        value_train, rej_train, wrong_train = calculate_value(logits_train, train_labels, t, Vr, Vc, Vw)\n",
        "\n",
        "        t_opt, cost_list = find_optimum_confidence_threshold(logits_val, y_val, confT_list, Vr, Vc, Vw)\n",
        "        value_test_opt, rej_test_opt, wrong_test_opt = calculate_value(logits_test, y_test, t_opt, Vr, Vc, Vw)\n",
        "        value_train_opt, rej_train_opt, wrong_train_opt  = calculate_value(logits_train, train_labels, t_opt, Vr, Vc, Vw)\n",
        "\n",
        "        t_opt_train, cost_list_ = find_optimum_confidence_threshold(logits_train, train_labels, confT_list, Vr, Vc, Vw)\n",
        "        t_opt_test, cost_list_ = find_optimum_confidence_threshold(logits_test, y_test, confT_list, Vr, Vc, Vw)\n",
        "        value_test_opt_test, rej_test_opt_test, wrong_test_opt_test = calculate_value(logits_test, y_test, t_opt_test, Vr, Vc, Vw)\n",
        "\n",
        "        rv_path = res_path + logfile_name + al_strategy + '_' + str(k) + \"_value.csv\"\n",
        "        with open(rv_path, 'w') as f:\n",
        "            c = 'batch, Vr, Vc, Vw, k, t_cal, t_opt_val, t_opt_train, t_opt_test, value_test, rej_test, wrong_test, value_train, rej_train, wrong_train, value_test_opt, rej_test_opt, wrong_test_opt, value_train_opt, rej_train_opt, wrong_train_opt, value_test_opt_test, rej_test_opt_test, wrong_test_opt_test'\n",
        "            f.write(c + '\\n')\n",
        "            res_i = '{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\\n'.format(0, Vr, Vc, Vw, k, t, t_opt, t_opt_train, t_opt_test, value_test, rej_test, wrong_test, value_train, rej_train, wrong_train, value_test_opt, rej_test_opt, wrong_test_opt, value_train_opt, rej_train_opt, wrong_train_opt, value_test_opt_test, rej_test_opt_test, wrong_test_opt_test)\n",
        "            f.write(res_i)\n",
        "\n",
        "    for Vw in Vw_list:\n",
        "        k = (-1)*(Vw / Vc)\n",
        "        rp_path = res_path + logfile_name + al_strategy + '_' + str(k) + \"_perf.csv\"\n",
        "        rv_path = res_path + logfile_name + al_strategy + '_' + str(k) + \"_value.csv\"\n",
        "\n",
        "        t = cost_based_threshold(k)\n",
        "\n",
        "        unlabeled_data = unlabeled_data_first.copy()\n",
        "        training_data = training_data_first.copy()\n",
        "        batchSize = alBatchSize\n",
        "\n",
        "        for alBatch in range(1, alBatchNum + 1, 1):\n",
        "            sampledIndices = []\n",
        "\n",
        "            unl_dataEmb = pool.poolDataEmb[poolData.index[poolData[iID].isin(unlabeled_data[iID].values)].tolist()]\n",
        "\n",
        "            if alBatch == alBatchNum:\n",
        "                batchSize = len(unlabeled_data[iID].values)\n",
        "                print(\"alBatchSize changed to: \", batchSize)\n",
        "            if al_strategy == 'tos':\n",
        "                idx = threshold_oriented_sampling(model, unl_dataEmb, batchSize, t)\n",
        "            else:\n",
        "                idx = tos_below(model, unl_dataEmb, batchSize, t)\n",
        "            sampledIndices = unlabeled_data.loc[idx][iID].tolist()\n",
        "            sampledSet = poolData.loc[poolData[iID].isin(sampledIndices)]\n",
        "            samplesDict[alBatch] = sampledIndices\n",
        "                    \n",
        "            training_data.reset_index(drop=True)\n",
        "            sampledSet.reset_index(drop=True)\n",
        "            training_data = pd.concat([training_data, sampledSet], axis=0).reset_index(drop=True)\n",
        "            training_data = training_data.sort_values(iID)\n",
        "            indices = unlabeled_data.loc[unlabeled_data[iID].isin(sampledIndices)].index.to_list()\n",
        "            unlabeled_data = unlabeled_data.drop(indices).reset_index(drop=True)\n",
        "            unlabeled_data = unlabeled_data.reset_index(drop=True)\n",
        "  \n",
        "            train_data = pool.poolDataEmb[poolData.index[poolData[iID].isin(training_data[iID].values)].tolist()]\n",
        "            train_labels = np.array(training_data[goldLabel].tolist())\n",
        "\n",
        "            model.fit(train_data, train_labels) \n",
        "\n",
        "            logits_train, logits_val, logits_test = evaluate_model(model, train_data, train_labels, poolDataEmb_val, y_val, poolDataEmb_test, y_test, rp_path, alBatch)\n",
        "\n",
        "            if alBatch in controlList2:\n",
        "                col = []\n",
        "                for i in range(logits_test.shape[1]):\n",
        "                    col.append(str(i))\n",
        "                df_lgt = pd.DataFrame(logits_test, columns = col)\n",
        "                df_lgt['y'] = y_test\n",
        "                df_lgt.to_csv(res_path + logfile_name + al_strategy + '_' + '_b_{}_k_{}_logits.csv'.format(alBatch, k))\n",
        "\n",
        "            t_opt, cost_list = find_optimum_confidence_threshold(logits_val, y_val, confT_list, Vr, Vc, Vw)\n",
        "\n",
        "            value_test, rej_test, wrong_test = calculate_value(logits_test, y_test, t, Vr, Vc, Vw)\n",
        "            value_train, rej_train, wrong_train = calculate_value(logits_train, train_labels, t, Vr, Vc, Vw)\n",
        "\n",
        "            value_test_opt, rej_test_opt, wrong_test_opt = calculate_value(logits_test, y_test, t_opt, Vr, Vc, Vw)\n",
        "            value_train_opt, rej_train_opt, wrong_train_opt  = calculate_value(logits_train, train_labels, t_opt, Vr, Vc, Vw)\n",
        "\n",
        "            t_opt_train, cost_list_ = find_optimum_confidence_threshold(logits_train, train_labels, confT_list, Vr, Vc, Vw)\n",
        "            t_opt_test, cost_list_ = find_optimum_confidence_threshold(logits_test, y_test, confT_list, Vr, Vc, Vw)\n",
        "            value_test_opt_test, rej_test_opt_test, wrong_test_opt_test = calculate_value(logits_test, y_test, t_opt_test, Vr, Vc, Vw)\n",
        "\n",
        "            with open(rv_path, 'a') as f:\n",
        "                res_i = '{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\\n'.format(alBatch, Vr, Vc, Vw, k, t, t_opt, t_opt_train, t_opt_test, value_test, rej_test, wrong_test, value_train, rej_train, wrong_train, value_test_opt, rej_test_opt, wrong_test_opt, value_train_opt, rej_train_opt, wrong_train_opt, value_test_opt_test, rej_test_opt_test, wrong_test_opt_test)\n",
        "                f.write(res_i)  \n",
        "      #  training_data.to_csv(res_path + logfile_name + al_strategy + \"_trainingData.csv\")\n",
        "        samplesDict_df = pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in samplesDict.items() ]))\n",
        "        samplesDict_df.to_csv(res_path + logfile_name + al_strategy + '_' + '_k_{}_sampledItems.csv'.format(k))"
      ],
      "metadata": {
        "id": "KKsrVBDttaGv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#load datasets\n",
        "for al_strategy in al_strategies:\n",
        "    pool = Data(encoder, data_folder + dataToTrain, 'train', encName)\n",
        "    mClass =  pool.mClass\n",
        "    pool.setStartState(minimum_training_items)\n",
        "\n",
        "    validation = Data(encoder, data_folder + dataToVal, 'val', encName)\n",
        "    validation_data = validation.data\n",
        "    y_val = np.array(validation_data[goldLabel].tolist())\n",
        "    test = Data(encoder, data_folder + dataToTest, 'test', encName)\n",
        "    test_data = test.data\n",
        "    y_test = np.array(test_data[goldLabel].tolist())\n",
        "\n",
        "    if al_strategy.startswith(\"tos\"):\n",
        "        run_tos(model,al_strategy,pool,mClass,validation,validation_data,y_val,test,test_data,y_test,alBatchSize,res_path,logfile_name,Vw_list)\n",
        "    else:\n",
        "        run_sota_AL(model,al_strategy,pool,mClass,validation,validation_data,y_val,test,test_data,y_test,alBatchSize,res_path,logfile_name,Vw_list)"
      ],
      "metadata": {
        "id": "-oIYW0d2redP"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}