PRI_2_14.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PRI Project, 2nd Delivery\n",
    "\n",
    "This implementation is the second part of our PRI project. We are Group 14, composed by: \n",
    "\n",
    "- Ana Evans nº 86379\n",
    "- Artur Guimaraes nº 86389\n",
    "- Francisco Rosa nº 86417\n",
    "\n",
    "\n",
    "This Notebook showcases the functional part of the second delivery. In each section we  present the tested function and a set of outputs. After each funtion we will mention the structure and the meaning of each input and output. Alternatively, you can include a standard function signature.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "jupyter": {
     "hide_code": true
    }
   },
   "outputs": [],
   "source": [
    "# --------------------------------\n",
    "# 2nd Delivery of the PRI project\n",
    "# 86379 - Ana Evans\n",
    "# 86389 - Artur Guimarães\n",
    "# 86417 - Francisco Rosa\n",
    "# --------------------------------\n",
    "import math\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os, os.path\n",
    "import pickle\n",
    "import re\n",
    "import scipy.sparse\n",
    "import shutil\n",
    "import sklearn\n",
    "import spacy\n",
    "import sys\n",
    "import time\n",
    "import whoosh\n",
    "\n",
    "from textblob import TextBlob\n",
    "from bs4 import BeautifulSoup\n",
    "from copy import deepcopy\n",
    "from heapq import nlargest\n",
    "from nltk import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "from rank_bm25 import BM25Okapi\n",
    "from scipy import stats\n",
    "\n",
    "from whoosh import index\n",
    "from whoosh import scoring\n",
    "from whoosh.fields import *\n",
    "from whoosh.qparser import *\n",
    "from whoosh.qparser import OrGroup\n",
    "from whoosh.qparser import QueryParser\n",
    "\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.metrics import *\n",
    "from sklearn.metrics.cluster import *\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.metrics.pairwise import euclidean_distances\n",
    "from sklearn.metrics.pairwise import manhattan_distances\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "# ______                  ___________   _____           _                 \n",
    "# \\ ___ \\                |_   _| ___ \\ /  ___|         | |                \n",
    "# | |_/ / __ _ ___  ___    | | | |_/ / \\ `--. _   _ ___| |_ ___ _ __ ___  \n",
    "# | ___ \\/ _` / __|/ _ \\   | | |    /   `--. \\ | | / __| __/ _ \\ '_ ` _ \\ \n",
    "# | |_/ / (_| \\__ \\  __/  _| |_| |\\ \\  /\\__/ / |_| \\__ \\ ||  __/ | | | | |\n",
    "# \\____/ \\__,_|___/\\___|  \\___/\\_| \\_| \\____/ \\__, |___/\\__\\___|_| |_| |_|\n",
    "#                                             __/ |                      \n",
    "#                                            |___/                       \n",
    "# --------------------------------------------------------------------------------\n",
    "\n",
    "topics = {}\n",
    "judged_documents = {}\n",
    "index_id = 1\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# write_to_file() - Small auxiliary function to write data to a file\n",
    "# -----------------------------------------------------------------------------\n",
    "def write_to_file(dic, filename):\n",
    "    with open('material/saved_data/{}.txt'.format(filename), 'wb') as write_f:\n",
    "        pickle.dump(dic, write_f)\n",
    "    return\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# read_from_file() - Small auxiliary function to read data from a file\n",
    "# -----------------------------------------------------------------------------\n",
    "def read_from_file(filename):\n",
    "    with open('material/saved_data/{}.txt'.format(filename), 'rb') as read_f:\n",
    "        return pickle.load(read_f) \n",
    "\n",
    "#--------------------------------------------------\n",
    "# get_xml_files_recursively - Auxiliary function to get_files_from_directory\n",
    "#\n",
    "# Input: path - The path to the parent directory or file from which to start our recursive function\n",
    "#               \n",
    "# Behaviour: Creates a list with the path to every file that's an hierarquical child of parent directory path,\n",
    "# recursively going through each child in Post-Order traversing\n",
    "#\n",
    "# Output: A List with the paths to each file child\n",
    "#--------------------------------------------------\n",
    "def get_xml_files_recursively(path, judged_documents, **kwargs):\n",
    "    files_list = []\n",
    "    directory_list = os.listdir(path)\n",
    "    for f in directory_list:\n",
    "        n_path = '{}{}/'.format(path,f)\n",
    "        if os.path.isdir(n_path):\n",
    "            files_list.extend(get_xml_files_recursively(n_path, judged_documents, **kwargs))\n",
    "\n",
    "        elif judged_documents != None:\n",
    "                if int(f.split('news')[0]) in judged_documents:\n",
    "                    files_list.append(re.sub('//','/','{}/{}'.format(path,f)))\n",
    "        else:\n",
    "            files_list.append(re.sub('//','/','{}/{}'.format(path,f)))\n",
    "    return files_list\n",
    "\n",
    "# -------------------------------------------------\n",
    "# get_files_from_directory - Recursively gets all files from directory or file path, parsing the files from xml to objects\n",
    "# and spliting them in D_Test and D_Train in the conditions specified by our project\n",
    "#\n",
    "# Input: path - The path to the parent directory or file from which to start our search\n",
    "#               \n",
    "# Behaviour: It starts by creating a list with the path to every file that's an hierarquical child of parent directory path,\n",
    "# recursively going through each child in Post-Order traversing. Afterwards it parses each and every file from xml to a runtime\n",
    "# object using the BeautifulSoup library. At last after having all files in object form it splits the dataset in D_Test and D_Train\n",
    "# sets, according to their identifier (D_Test -> identifier > 1996-09-30   D_Train -> identifier <= 1996-09-30)\n",
    "#\n",
    "# Output: A List with the Lists of file objects present in D_Test and D_Train\n",
    "# -------------------------------------------------\n",
    "def get_files_from_directory(path, judged_documents, **kwargs):\n",
    "    file_list = get_xml_files_recursively(path, judged_documents, **kwargs)\n",
    "\n",
    "    parsed_files_test = []\n",
    "    parsed_files_train = []\n",
    "\n",
    "    if 'set' in kwargs and kwargs['set'] == 'test':\n",
    "        for f in file_list:\n",
    "            date_identifier = int(f.split('/')[2])\n",
    "\n",
    "            if date_identifier <= 19960930:\n",
    "                continue\n",
    "\n",
    "            open_file = open(f, 'r')\n",
    "            parsed_file = BeautifulSoup(open_file.read(), 'lxml')\n",
    "            \n",
    "            if parsed_file.copyright != None:\n",
    "                parsed_file.copyright.decompose()\n",
    "\n",
    "            if parsed_file.codes != None:\n",
    "                parsed_file.codes.decompose()\n",
    "                \n",
    "            parsed_files_test += [parsed_file,]\n",
    "\n",
    "    elif 'set' in kwargs and kwargs['set'] == 'train':\n",
    "        for f in file_list:\n",
    "            date_identifier = int(f.split('/')[2])\n",
    "\n",
    "            if date_identifier > 19960930:\n",
    "                break\n",
    "\n",
    "            open_file = open(f, 'r')\n",
    "            parsed_file = BeautifulSoup(open_file.read(), 'lxml')\n",
    "            \n",
    "            if parsed_file.copyright != None:\n",
    "                parsed_file.copyright.decompose()\n",
    "\n",
    "            if parsed_file.codes != None:\n",
    "                parsed_file.codes.decompose()\n",
    "                \n",
    "            parsed_files_train += [parsed_file,]\n",
    "\n",
    "    else:\n",
    "        for f in file_list:\n",
    "            date_identifier = int(f.split('/')[2])\n",
    "\n",
    "            open_file = open(f, 'r')\n",
    "            parsed_file = BeautifulSoup(open_file.read(), 'lxml')\n",
    "            \n",
    "            if parsed_file.copyright != None:\n",
    "                parsed_file.copyright.decompose()\n",
    "\n",
    "            if parsed_file.codes != None:\n",
    "                parsed_file.codes.decompose()\n",
    "                \n",
    "            if date_identifier <= 19960930:\n",
    "                parsed_files_train += [parsed_file,]\n",
    "            else:\n",
    "                parsed_files_test += [parsed_file,]\n",
    "\n",
    "    return (parsed_files_test, parsed_files_train)\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------------------\n",
    "# processing - Processes text in String form\n",
    "#\n",
    "# Input: text - The text in String form to be processed\n",
    "#        **kwargs - Optional named arguments, with the following functionality (default values prefixed by *)\n",
    "#               lowercasing [*True | False]: Flag to perform Lowercasing \n",
    "#               punctuation [*True | False]: Flag to remove punction\n",
    "#               spellcheck [True | *False]: Flag to perform spell check using TextBlob\n",
    "#               stopwords [*True | False]: Flag to remove Stop Words \n",
    "#               simplication [*lemmatization | stemming | None]: Flag to perform Lemmatization or Stemming\n",
    "#               \n",
    "# Behaviour: Procceses the text in the input argument text as refered to by the arguments in **kwargs,\n",
    "# behaviour being completely dependent on them except for Tokenization which is always performed\n",
    "#\n",
    "# Output: A String with the processed text \n",
    "# ----------------------------------------------------------------------------------------------------------\n",
    "def processing(text, **kwargs):\n",
    "\n",
    "    p_text = text\n",
    "    # Lowercasing the entire string\n",
    "    if 'lowercasing' not in kwargs or kwargs['lowercasing']:\n",
    "        p_text = p_text.lower()\n",
    "\n",
    "    #Remove punctuation\n",
    "    if 'punctuation' not in kwargs or kwargs['punctuation']:\n",
    "        p_text = re.sub(\"[/-]\",\" \",p_text)\n",
    "        p_text = re.sub(\"[.,;:\\\"\\'!?`´()$£€]\",\"\",p_text)\n",
    "\n",
    "    # Spell Check\n",
    "    if \"spellcheck\" in kwargs and kwargs['spellcheck']:          \n",
    "        p_text = str(TextBlob(p_text).correct())\n",
    "\n",
    "    # Tokenization\n",
    "    tokens = nltk.word_tokenize(p_text)\n",
    "    string_tokens = ''\n",
    "\n",
    "    # Spell Check correction\n",
    "    if \"spellcheck\" in kwargs and kwargs['spellcheck']:\n",
    "        n_tokens = []\n",
    "        for word in tokens:           \n",
    "            n_tokens += ' {}'.format(TextBlob(word).correct)\n",
    "\n",
    "    # Lemmatization\n",
    "    if 'simplification' not in kwargs or kwargs['simplification'] == 'lemmatization':\n",
    "        lemma = WordNetLemmatizer()\n",
    "\n",
    "        #Remove stopwords\n",
    "        if 'stopwords' not in kwargs or kwargs['stopwords']:\n",
    "            for word in tokens:\n",
    "                if word not in stopwords.words('English'):   \n",
    "                    string_tokens += ' {}'.format(lemma.lemmatize(word))\n",
    "        else: \n",
    "            for word in tokens: \n",
    "                string_tokens += ' {}'.format(lemma.lemmatize(word))\n",
    "\n",
    "    # Stemming\n",
    "    elif kwargs['simplification'] == 'stemming':\n",
    "        stemer = nltk.stem.snowball.EnglishStemmer()\n",
    "\n",
    "        #Remove stopwords\n",
    "        if 'stopwords' not in kwargs or kwargs['stopwords']:\n",
    "            for word in tokens:\n",
    "                if word not in stopwords.words('English'):   \n",
    "                    string_tokens += ' {}'.format(stemer.stem(word))\n",
    "        else: \n",
    "            for word in tokens: \n",
    "                string_tokens += ' {}'.format(stemer.stem(word))\n",
    "\n",
    "    # Case for no simplification\n",
    "    else:\n",
    "        for word in tokens: \n",
    "            string_tokens += ' {}'.format(word)   \n",
    "\n",
    "    # Removing the first whitespace in the output \n",
    "    return string_tokens[1:]\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# process_collection - Small auxiliary function to externaly process a text \n",
    "# collection independently of program function\n",
    "# -----------------------------------------------------------------------------\n",
    "\n",
    "def process_collection(collection, tokenize, **kwargs):\n",
    "    result = {}\n",
    "    for doc in collection:\n",
    "        item_id = int(doc.newsitem.get('itemid'))\n",
    "        title = processing(re.sub('<[^<]+>', \"\", str(doc.title)), **kwargs)\n",
    "        dateline = processing(re.sub('<[^<]+>|\\w[0-9]+-[0-9]+-[0-9]+\\w', \"\", str(doc.dateline)), **kwargs)\n",
    "        text = processing(re.sub('<[^<]+>', \"\", str(doc.find_all('text')))[1:-1], **kwargs)\n",
    "        \n",
    "        if tokenize:\n",
    "            result[item_id] = nltk.word_tokenize('{} {} {}'.format(title, dateline, text))\n",
    "        else:\n",
    "            result[item_id] = '{}\\n{}\\n{}'.format(title, dateline, text)\n",
    "\n",
    "    return result\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "# tfidf_process - Processes our entire document collection with a tf-idf vectorizer \n",
    "# and transforms the entire collection into tf-idf spaced vectors \n",
    "#\n",
    "# Input: doc_dic - The entire document collection in dictionary form\n",
    "#        **kwargs - Optional parameters with the following functionality (default values prefixed by *)\n",
    "#               norm [*l2 | l1]: Method to calculate the norm of each output row\n",
    "#               min_df [*1 | float | int]: Ignore the terms which have a freq lower than min_df\n",
    "#               max_df [*1.0 | float | int]: Ignore the terms which have a freq higher than man_df\n",
    "#               max_features [*None | int]: \n",
    "#\n",
    "# Behaviour: Creates a tf-idf vectorizer and fits the entire document collection into it. \n",
    "# Afterwards, transforms the entire document collection into vector form, allowing it to be \n",
    "# directly used to calculate similarities. It also converts structures into to an easy form to manipulate \n",
    "# at the previous higher level.\n",
    "#\n",
    "# Output: The tf-idf vectorizer created, a list of document keys (ids) and the entire doc\n",
    "# collection in vector form.\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "def tfidf_process(doc_dic, **kwargs):\n",
    "    doc_keys = list(doc_dic.keys())\n",
    "    doc_list = []\n",
    "\n",
    "    for doc in doc_keys:\n",
    "        doc_list.append(doc_dic[doc])\n",
    "\n",
    "    norm = 'l2' if 'norm' not in kwargs else kwargs['norm']\n",
    "    min_df = 2 if 'min_df' not in kwargs else kwargs['min_df']\n",
    "    max_df = 0.8 if 'max_df' not in kwargs else kwargs['max_df']\n",
    "    max_features = None if 'max_features' not in kwargs else kwargs['max_features']\n",
    "    stop_words = None if 'remove_stopwords' not in kwargs else kwargs['remove_stopwords']\n",
    "\n",
    "    vec = TfidfVectorizer(norm=norm, min_df=min_df, max_df=max_df, max_features=max_features, stop_words= stop_words)\n",
    "    vec.fit(doc_list)\n",
    "\n",
    "    doc_list_vectors = vec.transform(doc_list)\n",
    "\n",
    "    return [vec, doc_keys, doc_list_vectors]\n",
    "\n",
    "# -----------------------------------------------------------------------\n",
    "# get_topics - Auxiliary function that gathers info on all topics\n",
    "#\n",
    "# Input: directory - Directory path for project materials\n",
    "# \n",
    "# Behaviour: Extracts topic info from '{directory}topics.txt' and updates\n",
    "# the global dictionary which stores topic info\n",
    "#\n",
    "# Output: None\n",
    "# -----------------------------------------------------------------------\n",
    "def get_topics(directory):\n",
    "    topics = {}\n",
    "    \n",
    "    topic_f = open('{}topics.txt'.format(directory), 'r')\n",
    "    parsed_file = BeautifulSoup(topic_f.read(), 'lxml')\n",
    "\n",
    "    topic_list = parsed_file.find_all('top')\n",
    "\n",
    "    for topic in topic_list:\n",
    "        split_topic = topic.getText().split('\\n')\n",
    "        split_topic = list(filter(lambda x: x!='', split_topic))\n",
    "\n",
    "        number = split_topic[0].split(' ')[2][1:]\n",
    "        title = processing(split_topic[1])\n",
    "        topics[int(number)] = re.sub(' +',' ',title)\n",
    "    return topics\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# get_R_set - Auxiliary function that extracts the R set\n",
    "#\n",
    "# Input: directory - Directory path for project materials\n",
    "# \n",
    "# Behaviour: Extracts the triplet (Topic id, Document id, Feedback) for each entry in the \n",
    "# R set, present in '{directory}qrels_test.txt' (R-test) and '{directory}qrels_test.txt' (R-train)\n",
    "#\n",
    "# Output: [R-Test, R-Train], each being a list of triplet entries\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def get_R_set(directory, **kwargs):\n",
    "    judged_documents = {}\n",
    "\n",
    "    r_test_f = open('{}qrels_test.txt'.format(directory), 'r')\n",
    "    r_train_f = open('{}qrels_train.txt'.format(directory), 'r')\n",
    "\n",
    "    r_test_lines = r_test_f.readlines()\n",
    "    r_train_lines = r_train_f.readlines()\n",
    "\n",
    "    r_test_lines = [r_test_lines, r_train_lines]\n",
    "    r_set = [{},{}]\n",
    "    \n",
    "    if 'index' in kwargs and kwargs['index'] == 'doc_id':\n",
    "        for i in range(2):\n",
    "            for line in r_test_lines[i]:\n",
    "                split_entry = line.split(' ')\n",
    "                topic_id = int(split_entry[0][1:])\n",
    "                doc_id = int(split_entry[1])\n",
    "\n",
    "                if doc_id not in judged_documents: \n",
    "                    judged_documents[doc_id] = True\n",
    "\n",
    "                feedback = int(split_entry[2])\n",
    "\n",
    "                if doc_id not in r_set[i]:\n",
    "                    r_set[i][doc_id] = {}\n",
    "                r_set[i][doc_id][topic_id] = feedback\n",
    "\n",
    "    else:\n",
    "        for i in range(2):\n",
    "            for line in r_test_lines[i]:\n",
    "                split_entry = line.split(' ')\n",
    "                topic_id = int(split_entry[0][1:])\n",
    "                doc_id = int(split_entry[1])\n",
    "\n",
    "                if doc_id not in judged_documents: \n",
    "                    judged_documents[doc_id] = True\n",
    "\n",
    "                feedback = int(split_entry[2])\n",
    "\n",
    "                if topic_id not in r_set[i]:\n",
    "                    r_set[i][topic_id] = {}\n",
    "                r_set[i][topic_id][doc_id] = feedback\n",
    "\n",
    "    return [r_set, judged_documents]\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# find_R_test_labels - Function that finds the test labels for a given R_Set\n",
    "#\n",
    "# Input: R_test - The R_Test set \n",
    "#\n",
    "# Behaviour: Extrapolates the feedback from the R_Test set to a dic or array\n",
    "#\n",
    "# Output: The R_Test set labels in dic or np array form\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def find_R_test_labels(R_test, **kwargs):\n",
    "    r_labels = None\n",
    "\n",
    "    if 'list' not in kwargs:\n",
    "        r_labels = {}\n",
    "        for doc in R_test:\n",
    "            r_labels[doc] = R_test[doc]\n",
    "\n",
    "    elif 'list' in kwargs and kwargs['list']:\n",
    "        r_labels = []\n",
    "        for doc in R_test:\n",
    "            r_labels.append(R_test[doc])\n",
    "\n",
    "    return r_labels\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# get_judged_docs - Small auxiliary function that returns the judged documents\n",
    "# in the given rcv1 directory\n",
    "# -----------------------------------------------------------------------------\n",
    "def get_judged_docs(material_dir, rcv1_dir):\n",
    "    judged_documents = get_R_set(material_dir)[1]\n",
    "    return get_files_from_directory(rcv1_dir, judged_documents, judged=True)\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# find_ranked_query_labels - Function that finds the test labels for given query_docs and r_labels\n",
    "#\n",
    "# Input: query_docs - The ranked query docs\n",
    "#        r_labels - the labels R_Test set produced \n",
    "#\n",
    "# Behaviour: Compares de R_Test set feedback with the ranked docs\n",
    "#\n",
    "# Output: The labels for the ranked query docs in np array form\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def find_ranked_query_labels(query_docs, r_labels):\n",
    "    q_docs = np.array(query_docs)\n",
    "    q_docs = q_docs[:,0]\n",
    "\n",
    "    query_labels = []\n",
    "    result_labels = []\n",
    "\n",
    "    for doc in query_docs:\n",
    "        if doc[0] in r_labels:\n",
    "            query_labels += [[doc[0], 1], ]\n",
    "            result_labels += [[doc[0], r_labels[doc[0]]], ]\n",
    "    \n",
    "    for doc in r_labels:\n",
    "        if doc not in q_docs:\n",
    "            query_labels += [[doc, 0], ]\n",
    "            result_labels += [[doc, r_labels[doc]], ]\n",
    "\n",
    "    return [np.array(query_labels), np.array(result_labels)]  \n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# find_boolean_query_labels - Function that finds the test labels for given query_docs and r_labels\n",
    "#\n",
    "# Input: query_docs - The query docs\n",
    "#        r_labels - the labels R_Test set produced \n",
    "#\n",
    "# Behaviour: Compares the R_Test set feedback with the ranked docs\n",
    "#\n",
    "# Output: The labels for the query docs in np array form\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def find_boolean_query_labels(query_docs, r_labels):\n",
    "    query_labels = []\n",
    "    result_labels = []\n",
    "\n",
    "    for doc in r_labels:\n",
    "        if doc in query_docs:\n",
    "            query_labels += [[doc, 1], ]\n",
    "            result_labels += [[doc, r_labels[doc]]]\n",
    "        else:\n",
    "            query_labels += [[doc, 0], ]\n",
    "            result_labels += [[doc, r_labels[doc]]]\n",
    "\n",
    "    return [np.array(query_labels), np.array(result_labels)]  \n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# normalize_dic() - Normalizes a dic \n",
    "# -----------------------------------------------------------------------------\n",
    "def normalize_dic(dic, **kwargs):\n",
    "    result_dic = {}\n",
    "\n",
    "    values = dic.values()\n",
    "    value_it = iter(values)\n",
    "\n",
    "    if type(next(value_it)) == str:\n",
    "        values_list = np.array([len(doc) for doc in values])\n",
    "    else:\n",
    "        values_list = np.array([score for score in values])\n",
    "    \n",
    "    if 'norm_method' not in kwargs or kwargs['norm_method'] == '1': \n",
    "        values_list = values_list / np.linalg.norm(values_list)\n",
    "\n",
    "    elif kwargs['norm_method'] == '2':\n",
    "        values_list = normalize(values_list[:,np.newaxis], axis=0).ravel()\n",
    "\n",
    "    elif kwargs['norm_method'] == 'zscore':\n",
    "        values_list = stats.zscore(values_list)\n",
    "\n",
    "    keys = list(dic.keys())\n",
    "    for i in range(len(keys)):\n",
    "        result_dic[keys[i]] = values_list[i]\n",
    "\n",
    "    return result_dic\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "# indexing - Creates an index after processing all text on data set D\n",
    "#\n",
    "# Input: D - The data set we will be building the index with\n",
    "#        **kwargs - Optional named arguments for text preprocessing, with the following functionality (default values prefixed by *)\n",
    "#               lowercasing [*True | False]: Flag to perform Lowercasing \n",
    "#               punctuation [*True | False]: Flag to remove punction\n",
    "#               spellcheck [True | *False]: Flag to perform spell check using TextBlob\n",
    "#               stopwords [*True | False]: Flag to remove Stop Words \n",
    "#               simplication [*lemmatization | stemming | None]: Flag to perform Lemmatization or Stemming\n",
    "#               \n",
    "# Behaviour: This function starts by creating the directory for our Index, after initializing our Schema fields. It then\n",
    "# processes all documents on data set D and stores valuable information from them on the index (identifier, title, dateline and text).\n",
    "# At last it commits the resulting processed documents to our index and calculates the total computational time the function used and the\n",
    "# Disk space required to store the index.\n",
    "#\n",
    "# Output: A triplet tuple with the Inverted Index in object structure, the computational time for the function and \n",
    "# the disk space required to store the Inverted Index \n",
    "# --------------------------------------------------------------------------------\n",
    "def indexing(D, **kwargs):\n",
    "    global index_id\n",
    "\n",
    "    start_time = time.time()\n",
    "    ind_name = 'index{}'.format(str(index_id))\n",
    "    ind_dir = '{}_dir'.format(ind_name)\n",
    "\n",
    "    if os.path.exists(ind_dir):\n",
    "        shutil.rmtree(ind_dir)\n",
    "        os.mkdir(ind_dir)\n",
    "    else:\n",
    "        os.mkdir(ind_dir)\n",
    "\n",
    "    schema = Schema(id= NUMERIC(stored=True), content= TEXT(stored=True))\n",
    "    ind = index.create_in(ind_dir, schema=schema, indexname=ind_name)\n",
    "    ind_writer = ind.writer()\n",
    "\n",
    "    if not index.exists_in(ind_dir, indexname=ind_name):\n",
    "        print(\"Error creating index\")\n",
    "        return\n",
    "\n",
    "    processed_docs = process_collection(D, True, **kwargs)\n",
    "\n",
    "    for doc in processed_docs:\n",
    "        ind_writer.add_document(id=int(doc), content=processed_docs[doc])\n",
    "\n",
    "    ind_writer.commit()\n",
    "    \n",
    "    time_required = round(time.time() - start_time, 6)\n",
    "    \n",
    "    space_required = os.path.getsize(ind_dir)\n",
    "\n",
    "    return (ind, time_required, space_required)\n",
    "\n",
    "# --------------------------------------------------------------------------------------------------------------------------------------\n",
    "# extract_topic_query - Return the top-k informative terms from the topic q agains I using parameterizable scoring\n",
    "#\n",
    "# Input: q - The identifier number of the topic we want to search about\n",
    "#        I - The Index object in which we will perform our search\n",
    "#        k - The number of top-k terms to return \n",
    "#        **kwargs - Optional named arguments to parameterize scoring, with the following functionality (default values prefixed by *)\n",
    "#               scoring [freq | tf-idf | dfree | pl2 |*bm25] - Chooses the scoring model we will use to score our terms\n",
    "#               C [float | *1.0] - Free parameter for the pl2 model\n",
    "#               B [float | *0.75] - Free parameter for the BM25 model\n",
    "#               content_B [float | *1.0] - Free parameter specific to the content field for the BM25 model\n",
    "#               k1 [float | *1.5] - Free parameter for the BM25 model\n",
    "#\n",
    "# Behaviour: Extracting the relevant model information from **kwargs, this function uses the index I present in its arguments \n",
    "# to perform a scored search on the top-k informative terms for topic q. It does so by creating a QueryParser object to parse\n",
    "# the entire lenght of terms from q we've stored in our global topics structure and by using searcher.key_terms() to return\n",
    "# the top terms according to our scoring weight vector. \n",
    "#\n",
    "# Output: A List that contains the top k terms \n",
    "# -----------------------------------------------------------------------------------------------------------------------------------------\n",
    "def extract_topic_query(q, I, k, **kwargs):\n",
    "    global topics \n",
    "    topic = topics[q]\n",
    "\n",
    "    topic_terms = []\n",
    "    weight_vector = None\n",
    "\n",
    "    # Chooses which score model to use from kwargs\n",
    "    if 'scoring' not in kwargs:\n",
    "        weight_vector = scoring.BM25F(B=0.75, content_B=1.0, K1=1.5)\n",
    "\n",
    "    elif kwargs['scoring'] == 'freq':\n",
    "        weight_vector = scoring.Frequency()\n",
    "\n",
    "    elif kwargs['scoring'] == 'tf-idf':\n",
    "        weight_vector = scoring.TF_IDF()\n",
    "\n",
    "    elif kwargs['scoring'] == 'dfree':\n",
    "        weight_vector = scoring.DFree()\n",
    "\n",
    "    elif kwargs['scoring'] == 'pl2':\n",
    "        C = 1.0 if 'C' not in kwargs else kwargs['C']\n",
    "\n",
    "        weight_vector = scoring.PL2(c=C)\n",
    "\n",
    "    elif kwargs['scoring'] == 'bm25':\n",
    "        b = 0.75 if 'B' not in kwargs else kwargs['B']\n",
    "        content_b = 1.0 if 'content_B' not in kwargs else kwargs['content_B']\n",
    "        k1 = 1.5 if 'K1' not in kwargs else kwargs['K1']\n",
    "\n",
    "        weight_vector = scoring.BM25F(B=b, content_B=content_b, K1=k1)    \n",
    "\n",
    "    with I.searcher(weighting=weight_vector) as searcher:\n",
    "        parser = QueryParser(\"content\", I.schema, group=OrGroup).parse(topic)\n",
    "        results = searcher.search(parser, limit=None)\n",
    "        res_list = [int(r.values()[1]) for r in results]\n",
    "\n",
    "        numbers_list = []\n",
    "        for i in res_list:\n",
    "            numbers_list += [searcher.document_number(id=i),]\n",
    "\n",
    "        topic_terms = searcher.key_terms(numbers_list, \"content\", numterms=k, normalize=True)\n",
    "      \n",
    "    result = []\n",
    "    for term in topic_terms:\n",
    "        result += [term[0], ]\n",
    "\n",
    "    return result\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------------------------\n",
    "# boolean_query_aux - Auxiliary function to boolean_query that will check repeated ocurrences of documents\n",
    "#\n",
    "# Input: document_lists - A List of Lists in which each inner List has all documents in which the n-th term appeared\n",
    "#        k - The number of terms we are using\n",
    "#\n",
    "# Behaviour: The function starts by calculating our error margin, in other words the number of missmatches a document\n",
    "# can have before we stop considering it as relevant. This function composes a very simple algorithmn, where for each\n",
    "# document we find in a sublist (non repeated, we use the list 'seen' to check that) we check if it's contained within \n",
    "# all other sublists, until it's not contained in miss_m + 1 lists. When that's the case, the document is no longer \n",
    "# relevant and we move on to the next one, iterating upon all elements of all sublists. The Time Complexity of this \n",
    "# function is O(N^2) while the Space Complexity is O(N)\n",
    "#\n",
    "# Output: A List of all relevants docs that don't exceed miss_m missmatches\n",
    "# -------------------------------------------------------------------------------------------------------------------\n",
    "def boolean_query_aux(document_lists, k):\n",
    "    miss_m = round(0.2*k)\n",
    "    seen = []\n",
    "    result_docs = []\n",
    "\n",
    "    for term_docs in document_lists:\n",
    "        for doc in term_docs:\n",
    "            if doc not in seen:\n",
    "                chances = miss_m\n",
    "                flag = True\n",
    "                for doc_list in document_lists:\n",
    "                    if doc not in doc_list:\n",
    "                        if chances == 0:\n",
    "                            flag = False\n",
    "                            break\n",
    "                        chances -= 1\n",
    "                if flag:\n",
    "                    result_docs += [doc,]\n",
    "                seen += [doc, ]\n",
    "\n",
    "    result_docs.sort()\n",
    "    return result_docs\n",
    "\n",
    "# ------------------------------------------------------------------------------------------\n",
    "# boolean_query - Function that will query all documents in index I and find those who contain\n",
    "# all top k-terms relevant to topic q allowing up to round(0.2*k) missmatches \n",
    "#\n",
    "# Input: q - The identifier number of the topic we want to search about \n",
    "#        k - The number of top k-terms to check documents for\n",
    "#        I - The Index object in which we will perform our search\n",
    "#\n",
    "# Behaviour: The function starts by running extract_topic_query to return top k-terms with which\n",
    "# we will search for the relevant docs for topic q. Then we use the index I to perform a simple\n",
    "# search on, parsing the result of our search per term to our auxiliary function. \n",
    "#\n",
    "# Output: A List of all relevants docs that don't exceed miss_m missmatches\n",
    "# ------------------------------------------------------------------------------------------\n",
    "def boolean_query(q, k, I, **kwargs):\n",
    "    terms = extract_topic_query(q, I, k, **kwargs)\n",
    "\n",
    "    document_lists = []\n",
    "    with I.searcher() as searcher:\n",
    "        for term in terms:\n",
    "            parser = QueryParser(\"content\", I.schema, group=OrGroup).parse(term)\n",
    "            results = searcher.search(parser, limit=None)\n",
    "            term_list = [int(r.values()[1]) for r in results]\n",
    "            document_lists += [term_list,]\n",
    "            \n",
    "    return boolean_query_aux(document_lists, k)\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------------------------------\n",
    "# cosine_scoring - Function that scores a document based on cosine similarity \n",
    "#\n",
    "# Input: searcher - The searcher associated with the index I\n",
    "#        all the other arguments are built-ins from FunctionWeighting() and old whoosh.scoring\n",
    "#        documentation\n",
    "#\n",
    "# Behaviour: Uses the tf-idf result from searcher.idf() and applies cosine similarity formula\n",
    "# to it\n",
    "#\n",
    "# Output: cosine similarity weight vector formula \n",
    "# ------------------------------------------------------------------------------------------\n",
    "def cosine_scoring(searcher, fieldnum, text, docnum, weight, QTF=1):\n",
    "    idf = searcher.idf(fieldnum, text)\n",
    "\n",
    "    DTW = (1.0 + math.log(weight)) * idf\n",
    "    QMF = 1.0\n",
    "    QTW = ((0.5 + (0.5 * QTF/ QMF))) * idf\n",
    "    return DTW * QTW\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# bpref - Function that runs the bpref evaluation metric\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def bpref(sol_labels):\n",
    "    R = 0\n",
    "    N = 0\n",
    "    bpref = 0\n",
    "    n_count = 0\n",
    "    for label in sol_labels:\n",
    "        if label == 0:\n",
    "            N += 1\n",
    "        else:\n",
    "            R += 1\n",
    "\n",
    "    for label in sol_labels:\n",
    "        if label == 0:\n",
    "            n_count += 1\n",
    "        else:\n",
    "            bpref += (1 - n_count/(min(R,N)))\n",
    "\n",
    "    return (1 / R) * bpref\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# reciprocal_rank_fusion - Auxiliary function to calculate the RRF for the top-p documents\n",
    "# Uses the formula RBF_score(f) = sum (1 / (50 + rank_f))\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def reciprocal_rank_fusion(p, ranking_lists):\n",
    "    document_ranks = {}\n",
    "\n",
    "    for rank_l in ranking_lists:\n",
    "        for i in range(len(rank_l )):\n",
    "            if rank_l[i][0] not in document_ranks:\n",
    "                document_ranks[rank_l[i][0]] = 0\n",
    "            document_ranks[rank_l[i][0]] += 1 / (50 + i+1)\n",
    "\n",
    "    p_highest = None\n",
    "\n",
    "    if p != None:\n",
    "        p_highest = nlargest(p, document_ranks, key=document_ranks.get)\n",
    "    else:\n",
    "        p_highest = nlargest(len(document_ranks), document_ranks, key=document_ranks.get)\n",
    "    \n",
    "    results = []\n",
    "\n",
    "    for p in p_highest:\n",
    "        results += [[p, document_ranks[p]]]  \n",
    "\n",
    "    return results\n",
    "\n",
    "# ------------------------------------------------------------------------------------------------\n",
    "# ranking - Function that will query all documents in index I and rank the top p ones\n",
    "#\n",
    "# Input: q - The identifier number of the topic we want to search about \n",
    "#        p - The number of top ranked documents we will return\n",
    "#        I - The Index object in which we will perform our search\n",
    "#        **kwargs - Optional named arguments to parameterize scoring, with the following functionality (default values prefixed by *)\n",
    "#               ranking [cosine | RRF | tf-idf | *bm25] - Chooses the scoring model we will use to score our terms\n",
    "#               B [float | *0.75] - Free parameter for the BM25 model\n",
    "#               content_B [float | *1.0] - Free parameter specific to the content field for the BM25 model\n",
    "#               k1 [float | *1.5] - Free parameter for the BM25 model\n",
    "#\n",
    "# Behaviour: The function uses the weight vector generated by its given scoring system to search and rank  \n",
    "# the top-p documents in the index according to the full topic text.\n",
    "#\n",
    "# Output: A List of lists that contains pairs [document_id, score] in descending score ordering\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def ranking(q, p, I, **kwargs):\n",
    "    global topics\n",
    "    topic = topics[q]\n",
    "\n",
    "    weight_vector = None\n",
    "    if 'ranking' not in kwargs:\n",
    "        weight_vector = scoring.BM25F(B=0.75, content_B=1.0, K1=1.5)\n",
    "\n",
    "    elif kwargs['ranking'] == 'cosine':\n",
    "        weight_vector = scoring.FunctionWeighting(cosine_scoring)\n",
    "\n",
    "    elif kwargs['ranking'] == 'tf-idf':\n",
    "        weight_vector = scoring.TF_IDF()\n",
    "\n",
    "    elif kwargs['ranking'] == 'bm25':\n",
    "        b = 0.75 if 'B' not in kwargs else kwargs['B']\n",
    "        content_b = 1.0 if 'content_B' not in kwargs else kwargs['content_B']\n",
    "        k1 = 1.5 if 'K1' not in kwargs else kwargs['K1']\n",
    "\n",
    "        weight_vector = scoring.BM25F(B=b, content_B=content_b, K1=k1)  \n",
    "\n",
    "    elif kwargs['ranking'] == 'RRF':\n",
    "        bm25_ranking_1 = ranking(q, p, I, ranking=\"bm25\")\n",
    "        bm25_ranking_2 = ranking(q, p, I, ranking=\"bm25\", b=0.5, content_b=1.25, k1=1.25)\n",
    "        bm25_ranking_3 = ranking(q, p, I, ranking=\"bm25\", b=0.5, content_b=1.5, k1=1.00)\n",
    "\n",
    "        return reciprocal_rank_fusion(p, [bm25_ranking_1, bm25_ranking_2, bm25_ranking_3])\n",
    "\n",
    "    with I.searcher(weighting=weight_vector) as searcher:\n",
    "        parser = QueryParser(\"content\", I.schema, group=OrGroup).parse(topic)\n",
    "        results = searcher.search(parser, limit=p)\n",
    "        \n",
    "        term_list = []\n",
    "\n",
    "        if p != None:\n",
    "            for i in range(p):\n",
    "                if i < len(results):\n",
    "                    term_list += [(int(results[i].values()[1]), results.score(i)), ]\n",
    "        else:\n",
    "            for i in range(len(results)):\n",
    "                term_list += [(int(results[i].values()[1]), results.score(i)), ]\n",
    "\n",
    "    return term_list\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# evaluate_ranked_query - Auxiliary function to calculate statistical data\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def evaluate_ranked_query(topic, o_labels, sol_labels, **kwargs):\n",
    "    results = {}\n",
    "\n",
    "    results['accuracy'] = accuracy_score(sol_labels, o_labels)\n",
    "    results['precision-micro'] = precision_score(sol_labels, o_labels, average='micro', zero_division=1)\n",
    "    results['precision-macro'] = precision_score(sol_labels, o_labels, average='macro', zero_division=1)\n",
    "    results['recall-micro'] =  recall_score(sol_labels, o_labels, average='micro')\n",
    "    results['recall-macro'] =  recall_score(sol_labels, o_labels, average='macro')\n",
    "    results['f-beta-micro'] = fbeta_score(sol_labels, o_labels, average='micro', beta=0.5)\n",
    "    results['f-beta-macro'] = fbeta_score(sol_labels, o_labels, average='macro', beta=0.5)\n",
    "    results['MAP'] = average_precision_score(sol_labels, o_labels)\n",
    "    results['BPREF'] = bpref(sol_labels)\n",
    "\n",
    "    if 'curves' in kwargs and kwargs['curves']:\n",
    "        precision, recall, _ = precision_recall_curve(sol_labels, o_labels)\n",
    "        PrecisionRecallDisplay(precision=precision, recall=recall).plot()\n",
    "        plt.title('Precision Recall curve for Ranked topic {}'.format(topic))\n",
    "        plt.show()\n",
    "\n",
    "    return results\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# evaluate_boolean_query - Auxiliary function to calculate statistical data\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def evaluate_boolean_query(topic, o_labels, sol_labels, **kwargs):\n",
    "    results = {}\n",
    "\n",
    "    results['accuracy'] = accuracy_score(sol_labels, o_labels)\n",
    "    results['precision-micro'] = precision_score(sol_labels, o_labels, average='micro', zero_division=1)\n",
    "    results['precision-macro'] = precision_score(sol_labels, o_labels, average='macro', zero_division=1)\n",
    "    results['recall-micro'] =  recall_score(sol_labels, o_labels, average='micro')\n",
    "    results['recall-macro'] =  recall_score(sol_labels, o_labels, average='macro')\n",
    "    results['f-beta-micro'] = fbeta_score(sol_labels, o_labels, average='micro', beta=0.5)\n",
    "    results['f-beta-macro'] = fbeta_score(sol_labels, o_labels, average='macro', beta=0.5)\n",
    "    results['MAP'] = average_precision_score(sol_labels, o_labels)\n",
    "\n",
    "    \n",
    "    if 'curves' in kwargs and kwargs['curves']:\n",
    "        precision, recall, _ = precision_recall_curve(sol_labels, o_labels)\n",
    "        PrecisionRecallDisplay(precision=precision, recall=recall).plot()\n",
    "        plt.title('Precision Recall curve for Boolean topic {}'.format(topic))\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "    return results\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# display_results_per_q - Auxiliary function to display calculated statistical data\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def display_results_per_q(q, results_ranked, results_boolean):\n",
    "    print(\"Result for search on Topic {}\".format(q))\n",
    "    print(\"\\nRanked Search:\")\n",
    "    for p in results_ranked:\n",
    "        result_str= ''\n",
    "        for m in results_ranked[p]:\n",
    "            result_str += '{} = {}, '.format(m, round(results_ranked[p][m],4)) \n",
    "        print(\"For p={}: {}\".format(p, result_str[:-2]))\n",
    "\n",
    "    print(\"\\nBoolean Search:\")\n",
    "    for k in results_boolean:\n",
    "        result_str= ''\n",
    "        for m in results_boolean[k]:\n",
    "            result_str += '{} = {}, '.format(m, round(results_boolean[k][m],4)) \n",
    "        print(\"For k={}: {}\".format(k, result_str[:-2]))\n",
    "\n",
    "        print(\"Result for search on Topic {}\".format(q))\n",
    "\n",
    "    return\n",
    "# -------------------------------------------------------------------------------------------------------\n",
    "# evaluation - Function that fully evaluates our IR model, providing full statiscal analysis for several\n",
    "# p and k values across multiple ranges and topics\n",
    "#\n",
    "# Input: Q_test - The set of topics we will evaluate the perform of our IR model on\n",
    "#        R_test - The topic labels we are looking for\n",
    "#        D_test - Our test set in collection form\n",
    "#        **kwargs - The additional args in this function also refer to the additional args in indexing(),\n",
    "#        ranking() and boolean_query(), for which documentation is provided above. Other than that, we have:\n",
    "#               k_range [list of ints | *[1,2,4,6,8,10]] - List of k values our model will test\n",
    "#               p_range [list of ints or None | *[100,200,300,400,500]] - List of p values our model will test\n",
    "#               curves [True | *False] - Display the precision/recall curves\n",
    "#\n",
    "# Behaviour: The function provides full statistics for every topic in Q_test, using R_test and D_test\n",
    "# to build an index. Then, for each p in p_range it will use ranking() to rank the top p documents\n",
    "# and for each k in k_range it will use k to evaluate the relevant docs using boolean_query(). In the end,\n",
    "# it uses retrival results to provide full statiscal analysis.\n",
    "#\n",
    "# Output: Full statistical analysis for the provided input args\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "def evaluation(Q_test, R_test, D_test, **kwargs):\n",
    "    # Standard index execution\n",
    "    I = indexing(D_test, **kwargs)[0]\n",
    "\n",
    "    results_ranked = {}\n",
    "    results_boolean = {}\n",
    "    k_range = [1,2,4,6,8,10] if 'k_range' not in kwargs else kwargs['k_range']\n",
    "    p_range = [100,200,300,400,500, None] if 'p_range' not in kwargs else kwargs['p_range']\n",
    "\n",
    "\n",
    "    for q in Q_test:\n",
    "        r_labels = find_R_test_labels(R_test[q])\n",
    "\n",
    "        for p in p_range:\n",
    "            score_docs = ranking(q, p, I, **kwargs)\n",
    "            ranked_labels = find_ranked_query_labels(score_docs, r_labels)\n",
    "\n",
    "            results_ranked[p] = evaluate_ranked_query(q, ranked_labels[0][:, 1],ranked_labels[1][:, 1], **kwargs)\n",
    "\n",
    "        for k in k_range:\n",
    "            boolean_docs = boolean_query(q, k, I, **kwargs)\n",
    "            query_labels = find_boolean_query_labels(boolean_docs, r_labels)\n",
    "\n",
    "            results_boolean[k] = evaluate_boolean_query(q, query_labels[0][:, 1], query_labels[1][:, 1], **kwargs)\n",
    "\n",
    "    display_results_per_q(q, results_ranked, results_boolean)\n",
    "    return\n",
    "\n",
    "# --------------------------------------------------------------------------------------------\n",
    "# overlapping_terms() - Function that finds the overlapping terms for a given k range\n",
    "#\n",
    "# Input: \n",
    "#\n",
    "# Behaviour: Queries the top terms for all k's in a given k range and checks them for overlap\n",
    "#\n",
    "# Output: Data about the overlaping terms\n",
    "# --------------------------------------------------------------------------------------------\n",
    "def overlapping_terms():\n",
    "    # Standard index execution\n",
    "    I = index.open_dir(\"index_judged_docs_dir\", indexname='index_judged_docs')\n",
    "\n",
    "    k_range = [2,3,5,7,10,15]\n",
    "\n",
    "    for k in k_range:\n",
    "        top_terms = {}\n",
    "        for q in range(101,201,1):\n",
    "            results = extract_topic_query(q, I, k)\n",
    "            for r in results:\n",
    "                if r not in top_terms:\n",
    "                    top_terms[r] = 0\n",
    "                top_terms[r] += 1\n",
    "\n",
    "        r_terms = 0\n",
    "        for term in top_terms:\n",
    "            if top_terms[term] > 1:\n",
    "                r_terms += 1\n",
    "        print(\"\\nNumber of overlapping terms: {}\".format(r_terms))\n",
    "        print(\"Percent of overlapping terms: {}%\".format(round(r_terms/len(top_terms)*100,3)))\n",
    "        print(top_terms)\n",
    "    return\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "#  _____  _              _               _               \n",
    "# /  __ \\| |            | |             (_)              \n",
    "# | /  \\/| | _   _  ___ | |_  ___  _ __  _  _ __    __ _ \n",
    "# | |    | || | | |/ __|| __|/ _ \\| '__|| || '_ \\  / _` |\n",
    "# | \\__/\\| || |_| |\\__ \\| |_|  __/| |   | || | | || (_| |\n",
    "# \\____/ |_| \\__,_||___/ \\__|\\___||_|   |_||_| |_| \\__, |\n",
    "#                                                   __/ |\n",
    "#                                                  |___/            \n",
    "# --------------------------------------------------------------------------------\n",
    "\n",
    "tfidf_vec_info = [None, None, None]\n",
    "Labels_pred = []\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "# get_clustering_score() - Gets a clustering methods score based on supervised, unsupervised\n",
    "# or mixed metrics. Uses a linear combination of all results.\n",
    "# --------------------------------------------------------------------------------\n",
    "def get_clustering_score(x, labels_true, labels_pred, target):\n",
    "    unsupervised_methods = { 'sil_score' : silhouette_score, 'calin_score' : calinski_harabasz_score, 'davies_score' : davies_bouldin_score}\n",
    "\n",
    "    supervised_methods = {'rands_score' : adjusted_rand_score, 'v_score' : v_measure_score, 'complete_score' : completeness_score, \n",
    "                          'fowlkes_score' : fowlkes_mallows_score, 'homogenity_score': homogeneity_score, 'mutual_score' : mutual_info_score}\n",
    "\n",
    "    if target == 'supervised':\n",
    "        unsupervised_methods = {}\n",
    "    elif target == 'unsupervised':\n",
    "        supervised_methods = {}\n",
    "\n",
    "    result = 0\n",
    "    for method in unsupervised_methods:\n",
    "        result += unsupervised_methods[method](x, labels_pred)\n",
    "\n",
    "    for method in supervised_methods:\n",
    "        score = supervised_methods[method](labels_true, labels_pred)\n",
    "        result += score\n",
    "        print('{} = {}'.format(method, score))\n",
    "\n",
    "    return result / (len(unsupervised_methods) + len(supervised_methods))\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "# trainsKmeans - trains the KMeans algorithm\n",
    "#\n",
    "# Input: vec_D - the set of document or topics to be clustered\n",
    "#       y - the set of ids and relevance info\n",
    "#       clusters - the list with the number of clusters to be attempted\n",
    "#       distance - parameter for the evaluation measures\n",
    "# \n",
    "# Behaviour: creates the KMeans clusters for the number of clusters previously\n",
    "# defined, and then applies a set of evaluation measures to select the best one\n",
    "#\n",
    "# Output: A list containing te best number of clusters to use, the list of labels\n",
    "# and the rands score\n",
    "# --------------------------------------------------------------------------------\n",
    "def trainKmeans(vec_D, y, clusters, distance):\n",
    "\n",
    "    array_D = vec_D.toarray()\n",
    "    best_result = [None, None, 0, 0]\n",
    "    for i in clusters:\n",
    "        clustering_kmeans = KMeans(n_clusters=i).fit(vec_D)\n",
    "        labels_pred = clustering_kmeans.labels_\n",
    "\n",
    "        score_mean = get_clustering_score(array_D, y, labels_pred, 'unsupervised')\n",
    "\n",
    "        if score_mean > best_result[2]:\n",
    "            best_result = [clustering_kmeans, labels_pred, score_mean, i]  \n",
    "\n",
    "    return best_result\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "# trainsAgglomerative - trains the Agglomerative Clustering algorithm\n",
    "#\n",
    "# Input: vec_D - the set of document or topics to be clustered\n",
    "#       y - the set of ids and relevance info\n",
    "#       clusters - the list with the number of clusters to be attempted\n",
    "#       distance - parameter for the evaluation measures\n",
    "# \n",
    "# Behaviour: creates the Agglomerative clusters for the number of clusters previously\n",
    "# defined, and then applies a set of evaluation measures to select the best one\n",
    "#\n",
    "# Output: A list containing te best number of clusters to use, the list of labels\n",
    "# and the rands score\n",
    "# --------------------------------------------------------------------------------\n",
    "def trainAgglomerative(vec_D, y, clusters, distance):\n",
    "\n",
    "    vec_D = vec_D.toarray()\n",
    "    best_result = [None, None, 0, 0]\n",
    "    for i in clusters:\n",
    "        clustering_agg = AgglomerativeClustering(n_clusters=i).fit(vec_D)\n",
    "        labels_pred = clustering_agg.labels_\n",
    "\n",
    "        score_mean = get_clustering_score(vec_D, y, labels_pred, 'unsupervised')\n",
    "\n",
    "        if score_mean > best_result[2]:\n",
    "            best_result = [clustering_agg, labels_pred, score_mean, i]\n",
    "\n",
    "    return best_result\n",
    "\n",
    "# ----------------------------------------------------------------------------------------------------\n",
    "# get_topic_subset() - Retrieves topics from the global variable topics that are contained within\n",
    "# Q \n",
    "# ----------------------------------------------------------------------------------------------------\n",
    "def get_topic_subset(q_test):\n",
    "    result = {}\n",
    "    for topic in topics:\n",
    "        if topic in q_test:\n",
    "            result[topic] = topics[topic]\n",
    "    return result\n",
    "\n",
    "# ----------------------------------------------------------------------------------------------------\n",
    "# get_category_ids() - Helper function to get category id's from collection D to serve as ground\n",
    "# truth\n",
    "# ----------------------------------------------------------------------------------------------------\n",
    "def get_category_ids(doc_keys):\n",
    "    clustered_docs = {}\n",
    "    for doc in doc_keys:\n",
    "        clustered_docs[doc] = True\n",
    "\n",
    "    D = get_files_from_directory('../rcv1/', clustered_docs)\n",
    "    D[0].extend(D[1])\n",
    "    D = D[0]\n",
    "\n",
    "    codes_y = []\n",
    "    for doc in D:\n",
    "        codes_y.append([])\n",
    "        if len(doc.find_all('codes')) > 1:\n",
    "            topic_section = doc.find_all('codes')[1]\n",
    "            topic_ids = topic_section.find_all('code')\n",
    "\n",
    "            for iden in topic_ids:\n",
    "                codes_y[-1].append(iden['code'])\n",
    "    codes_y = np.array(codes_y, dtype=object)\n",
    "\n",
    "    return codes_y\n",
    "\n",
    "# --------------------------------------------------------------------------------\n",
    "#  _____                                   _                 _ \n",
    "# /  ___|                                 (_)               | |\n",
    "# \\ `--.  _   _  _ __    ___  _ __ __   __ _  ___   ___   __| |\n",
    "#  `--. \\| | | || '_ \\  / _ \\| '__|\\ \\ / /| |/ __| / _ \\ / _` |\n",
    "# /\\__/ /| |_| || |_) ||  __/| |    \\ V / | |\\__ \\|  __/| (_| |\n",
    "# \\____/  \\__,_|| .__/  \\___||_|     \\_/  |_||___/ \\___| \\__,_|\n",
    "#               | |                                            \n",
    "#               |_|                                             \n",
    "# --------------------------------------------------------------------------------\n",
    "\n",
    "d_train = {}\n",
    "d_test = {}\n",
    "r_train = {}\n",
    "r_test = {}\n",
    "topic_vectorizers = {}\n",
    "\n",
    "best_params = []\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "# create_vectorizer - Processes our entire document collection with a vectorizer \n",
    "# and transforms the entire collection into spaced vectors \n",
    "#\n",
    "# Input: doc_dic - The entire document collection in dictionary form\n",
    "#        feature_space - String defining which vectorizer is used (tf, idf or tf-idf)\n",
    "#        **kwargs - Optional parameters with the following functionality (default values prefixed by *)\n",
    "#               norm [*l2 | l1]: Method to calculate the norm of each output row\n",
    "#               min_df [*1 | float | int]: Ignore the terms which have a freq lower than min_df\n",
    "#               max_df [*1.0 | float | int]: Ignore the terms which have a freq higher than man_df\n",
    "#               max_features [*None | int]: \n",
    "#\n",
    "# Behaviour: Creates a vectorizer (tf, idf or tf-idf) and fits the entire document collection into it. \n",
    "# Afterwards, transforms the entire document collection into vector form, allowing it to be \n",
    "# directly used to calculate similarities. It also converts structures into to an easy form to manipulate \n",
    "# at the previous higher level.\n",
    "#\n",
    "# Output: The created Vectorizer and the entire doc collection in vector form.\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "\n",
    "def create_vectorizer(doc_dic, feature_space, **kwargs):\n",
    "    doc_keys = list(doc_dic.keys())\n",
    "    doc_list = []\n",
    "\n",
    "    for doc in doc_keys:\n",
    "        doc_list.append(doc_dic[doc])\n",
    "\n",
    "    norm = 'l2' if 'norm' not in kwargs else kwargs['norm']\n",
    "    min_df = 2 if 'min_df' not in kwargs else kwargs['min_df']\n",
    "    max_df = 0.8 if 'max_df' not in kwargs else kwargs['max_df']\n",
    "    max_features = None if 'max_features' not in kwargs else kwargs['max_features']\n",
    "    stop_words = 'english' if 'remove_stopwords' not in kwargs else kwargs['remove_stopwords']\n",
    "    vec = None\n",
    "    doc_list_vectors = None\n",
    "\n",
    "    if feature_space == 'tf':\n",
    "        vec = CountVectorizer(min_df=min_df, max_df=max_df, max_features=max_features, stop_words= stop_words)\n",
    "        vec.fit(doc_list)\n",
    "        vec.transform(doc_list)\n",
    "        \n",
    "    elif feature_space == 'idf':\n",
    "        vec = []\n",
    "        vec.append(CountVectorizer(min_df=min_df, max_df=max_df, max_features=max_features, stop_words= stop_words))\n",
    "        aux_vectors = vec[0].fit_transform(doc_list)\n",
    "\n",
    "        vec.append(TfidfTransformer(smooth_idf=True, use_idf=True)) \n",
    "        doc_list_vectors = vec[1].fit_transform(aux_vectors)\n",
    "\n",
    "    elif feature_space == 'tf-idf':\n",
    "        vec = TfidfVectorizer(norm=norm, min_df=min_df, max_df=max_df, max_features=max_features, stop_words= stop_words)\n",
    "\n",
    "    if type(vec) != list:\n",
    "        vec.fit(doc_list)\n",
    "        doc_list_vectors = vec.transform(doc_list)\n",
    "\n",
    "    return [vec, doc_list_vectors]\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# display_results - Auxiliary function to display calculated statistical data\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def display_results(q, results):\n",
    "    print(\"Result for search on Topic {}\".format(q))\n",
    "    result_str= ''\n",
    "    for m in results:\n",
    "        if m != 'ranking':\n",
    "            result_str += '{} = {}, '.format(m, round(results[m],4)) \n",
    "    print(\"{}\\n\".format(result_str[:-2]))\n",
    "  \n",
    "    if 'ranking' in results:\n",
    "        print(\"Top ranked documents on Topic {}\".format(q))\n",
    "        print(\"{}\".format(results['ranking']))\n",
    "\n",
    "    return\n",
    "# --------------------------------------------------------------------------------\n",
    "#  _____                     _      ______               _     _               \n",
    "# |  __ \\                   | |     | ___ \\             | |   (_)              \n",
    "# | |  \\/ _ __  __ _  _ __  | |__   | |_/ / __ _  _ __  | | __ _  _ __    __ _ \n",
    "# | | __ | '__|/ _` || '_ \\ | '_ \\  |    / / _` || '_ \\ | |/ /| || '_ \\  / _` |\n",
    "# | |_\\ \\| |  | (_| || |_) || | | | | |\\ \\| (_| || | | ||   < | || | | || (_| |\n",
    "# \\____/|_|   \\__,_|| .__/ |_| |_| \\_| \\_|\\__,_||_| |_||_|\\_\\|_||_| |_| \\__, |\n",
    "#                   | |                                                  __/ |\n",
    "#                   |_|                                                 |___/                       \n",
    "# --------------------------------------------------------------------------------\n",
    "\n",
    "# -------------------------------------------------------------------------------------\n",
    "# manhattan_distance_dic - Computes the manhattan distance between a document\n",
    "# and a given list of documents\n",
    "#\n",
    "# Input: doc_query - Document that serves as basis to compare all other documents to\n",
    "#        vectorizer - The structure that contains our tf-idf vectorizer\n",
    "#        doc_keys - A list of all document keys\n",
    "#        doc_vectors - All documents in vector form contained in the vectorizer space\n",
    "#        theta - The similarity threshold \n",
    "#\n",
    "# Behaviour: It starts by transforming the query document into its vector notion in the\n",
    "# vectorizer space. Afterwards, it calculates pairwise similarity based on the inverse \n",
    "# manhattan distance between all document vectors. In the end returns a dictionary with \n",
    "# all documents that have their similarity values (1/distance) greater than or equal to theta.\n",
    "#\n",
    "# Output: Dictionary with all documents that pass the similarity treshold\n",
    "# -------------------------------------------------------------------------------------\n",
    "def manhattan_distance_dic(doc_query, vectorizer, doc_keys, doc_vectors, theta, **kwargs):\n",
    "    result = {}\n",
    "\n",
    "    doc_vector = vectorizer.transform(doc_query)\n",
    "    distance_list = manhattan_distances(doc_vector, doc_vectors)[0]\n",
    "\n",
    "    for i in range(len(distance_list)):\n",
    "        if distance_list[i] != 0 and 1/distance_list[i] >= theta:\n",
    "            result[int(doc_keys[i])] = 1/distance_list[i]\n",
    "\n",
    "    return result\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# eucledian_distance_dic -  Computes the eucledian distance between a document\n",
    "# and a given list of documents\n",
    "#\n",
    "# Input: doc_query - Document that serves as basis to compare all other documents to\n",
    "#        vectorizer - The structure that contains our tf-idf vectorizer\n",
    "#        doc_keys - A list of all document keys\n",
    "#        doc_vectors - All documents in vector form contained in the vectorizer space\n",
    "#        theta - The similarity threshold \n",
    "#\n",
    "# Behaviour: It starts by transforming the query document into its vector notion in the\n",
    "# vectorizer space. Afterwards, it calculates pairwise similarity based on the inverse \n",
    "# eucledian distance between all document vectors.  In the end returns a dictionary with \n",
    "# all documents that have their similarity values (1/distance) greater than or equal to theta.\n",
    "#\n",
    "# Output: Dictionary with all documents that pass the similarity treshold\n",
    "# -----------------------------------------------------------------------------\n",
    "def eucledian_distance_dic(doc_query, vectorizer, doc_keys, doc_vectors, theta, **kwargs):\n",
    "    result = {}\n",
    "\n",
    "    doc_vector = vectorizer.transform(doc_query)\n",
    "    distance_list = euclidean_distances(doc_vector, doc_vectors)[0]\n",
    "\n",
    "    for i in range(len(distance_list)):\n",
    "        if distance_list[i] != 0 and 1/distance_list[i] >= theta:\n",
    "            result[int(doc_keys[i])] = 1/distance_list[i]\n",
    "\n",
    "    return result\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# cosine_similarity_dic - Computes the cosine similarity between a document\n",
    "# and a given list of documents\n",
    "#\n",
    "# Input: doc_query - Document that serves as basis to compare all other documents to\n",
    "#        vectorizer - The structure that contains our tf-idf vectorizer\n",
    "#        doc_keys - A list of all document keys\n",
    "#        doc_vectors - All documents in vector form contained in the vectorizer space\n",
    "#        theta - The similarity threshold \n",
    "#\n",
    "# Behaviour: It starts by transforming the query document into its vector notion in the\n",
    "# vectorizer space. Afterwards, it calculates pairwise similarity based on the cosine \n",
    "# similarity measure between all document vectors. In the end returns a dictionary with \n",
    "# all documents that have their similarity values greater than or equal to theta.\n",
    "#\n",
    "# Output: Dictionary with all documents that pass the similarity treshold\n",
    "# -----------------------------------------------------------------------------\n",
    "def cosine_similarity_dic(doc_query, vectorizer, doc_keys, doc_vectors, theta, **kwargs):\n",
    "    result = {}\n",
    "\n",
    "    doc_vector = vectorizer.transform(doc_query)\n",
    "    distance_list = cosine_similarity(doc_vector, doc_vectors)[0]\n",
    "\n",
    "    for i in range(len(distance_list)):\n",
    "        if distance_list[i] >= theta:\n",
    "            result[int(doc_keys[i])] = distance_list[i]\n",
    "\n",
    "    return result\n",
    "\n",
    "# ------------------------------------------------------------------------------\n",
    "# sim_method_helper - Short helper method to encapsulate choosing the correct\n",
    "# similarity function\n",
    "#\n",
    "# Input: sim - The string which represents which similarity function to use\n",
    "#\n",
    "# Behaviour: Matches the string with a dictionary of functions we have available\n",
    "#\n",
    "# Output: The function to use\n",
    "# ------------------------------------------------------------------------------\n",
    "def sim_method_helper(sim):\n",
    "    sim_methods = {'cosine': cosine_similarity_dic, 'euclidean': eucledian_distance_dic, 'manhattan': manhattan_distance_dic}\n",
    "    sim_method = None\n",
    "    \n",
    "    if sim == None:\n",
    "        sim_method = sim_methods['cosine']\n",
    "\n",
    "    elif sim == 'cosine' or sim == 'euclidean' or sim == 'manhattan':\n",
    "        sim_method = sim_methods[sim]\n",
    "    \n",
    "    else:\n",
    "        print(\"Error: similarity measure not recognized.\")\n",
    "\n",
    "    return sim_method\n",
    "\n",
    "# ------------------------------------------------------------------------------------------------\n",
    "# ranking_for_page_rank - Function that uses our ranking function to format data for page_rank\n",
    "# non-uniform priors\n",
    "#\n",
    "# Input: query - The query we are searching our index on \n",
    "#        p - The number of top ranked documents we will return\n",
    "#        D - A document collection \n",
    "#        **kwargs - Optional named arguments to parameterize scoring, with the following functionality (default values prefixed by *)\n",
    "#               method [*None | len ] - Chooses a method to calculate priors\n",
    "#               \n",
    "# Behaviour: Uses or original IR system or the documents lenght to calculate non-uniform priors\n",
    "#\n",
    "# Output: A dictionary with the top p entries in the form doc_id : score\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def ranking_page_rank(query, p, D, **kwargs):\n",
    "    from base_IRsystem import indexing\n",
    "    result_dic = {}\n",
    "\n",
    "    if 'prior_method' not in kwargs:\n",
    "        I = indexing(D, **kwargs)[0]\n",
    "        \n",
    "\n",
    "        with I.searcher(weighting=scoring.BM25F(B=0.75, content_B=1.0, K1=1.5)) as searcher:\n",
    "            parser = QueryParser(\"content\", I.schema, group=OrGroup).parse(query)\n",
    "            results = searcher.search(parser, limit=p)\n",
    "\n",
    "            if p != None:\n",
    "                for i in range(p):\n",
    "                    if i < len(results):\n",
    "                        result_dic[int(results[i].values()[1])] = results.score(i)\n",
    "    \n",
    "    elif kwargs['prior_method'] == 'len':\n",
    "        processed_collection = process_collection(D, False)\n",
    "        result_dic = processed_collection \n",
    "\n",
    "    return normalize_dic(result_dic)\n",
    "\n",
    "# ---------------------------------------------------------------------------------------------------------\n",
    "# page_rank - Function that directly uses the Page Rank algorithm with a \n",
    "# variation for undirected graphs and uses it to calculate a score for each\n",
    "# candidate based on the provided link_graph. \n",
    "#\n",
    "# Input: link_graph - The undirected graph that contains all document links\n",
    "#        and their correspondent weight.\n",
    "#        q - A topic query in the form of topic identifier (int)\n",
    "#        D - The document collection we built our graph with\n",
    "#        **kwargs - Optional parameters with the following functionality (default values prefixed by *)\n",
    "#               iter [*50 | int]: Number of iterations to run the algorithm in \n",
    "#               p [*0.15 | float]: Starting p value which represents the residual probability for each node\n",
    "#               prior [*uniform | non-uniform]: Method to calculate priors in our algorithm \n",
    "# \n",
    "# Behaviour: This function starts by setting the default values for the Page Rank algorithm, and after \n",
    "# selecting which prior to use, it applies the algorithm max_iters number of times. It also builds some\n",
    "# auxiliary structures like link_count to ensure we don't repeatly calculate const values. \n",
    "#\n",
    "# Output: The resulting PageRank graph in dictionary form. \n",
    "# ---------------------------------------------------------------------------------------------------------\n",
    "def page_rank(link_graph, q, D, **kwargs):\n",
    "    result_graph = {}\n",
    "\n",
    "    max_iters = 50 if 'iter' not in kwargs else kwargs['iter']\n",
    "    p = 0.15 if 'p' not in kwargs else kwargs['p']\n",
    "    N = len(link_graph)\n",
    "    follow_p = 1 - p\n",
    "    prior = None\n",
    "\n",
    "    if 'prior' not in kwargs or kwargs['prior'] == 'uniform':\n",
    "        prior = p / N\n",
    "\n",
    "        # Setting uniform priors\n",
    "        for doc in link_graph:\n",
    "            result_graph[doc] = prior\n",
    "\n",
    "        # Dictionary to save max_iters * (N-1) len() operations\n",
    "        link_count = {}\n",
    "        for doc in link_graph:\n",
    "            link_count[doc] = len(link_graph[doc])\n",
    "\n",
    "        for _ in range(max_iters):\n",
    "            iter_graph = {}\n",
    "\n",
    "            for doc in result_graph:\n",
    "                cumulative_post = 0\n",
    "\n",
    "                for link in link_graph[doc]:\n",
    "                    cumulative_post += (result_graph[link] / link_count[doc]) \n",
    "                iter_graph[doc] = prior + follow_p * cumulative_post \n",
    "\n",
    "            result_graph = iter_graph\n",
    "\n",
    "    elif 'prior' in kwargs and kwargs['prior'] == 'non-uniform':\n",
    "\n",
    "        ranked_dic = ranking_page_rank(topics[q], len(link_graph), D, **kwargs)\n",
    "        prior_dic = {}\n",
    "\n",
    "        # Initialize prior using original IR system\n",
    "        for doc in link_graph:\n",
    "            if doc in ranked_dic:\n",
    "                prior_dic[doc] = ranked_dic[doc]\n",
    "            else:\n",
    "                prior_dic[doc] = 0\n",
    "\n",
    "        result_graph = deepcopy(prior_dic)\n",
    "\n",
    "        # Dictionary to save max_iters * (N-1) cum_sum operations\n",
    "        link_weighted_count = {}\n",
    "        for doc in link_graph:\n",
    "            link_weighted_count[doc] = 0\n",
    "            for link in link_graph[doc]:\n",
    "                link_weighted_count[doc] += link_graph[doc][link]\n",
    "\n",
    "        for _ in range(max_iters):\n",
    "            iter_graph = {}\n",
    "\n",
    "            for doc in result_graph:\n",
    "                cumulative_prior = 0\n",
    "                cumulative_post = 0\n",
    "\n",
    "                for link in link_graph[doc]:\n",
    "                    cumulative_prior += prior_dic[link]\n",
    "                    cumulative_post += ((result_graph[link] * link_graph[link][doc]) / link_weighted_count[doc]) \n",
    "\n",
    "                iter_graph[doc] = p * cumulative_prior + follow_p * cumulative_post \n",
    "\n",
    "            result_graph = iter_graph\n",
    "\n",
    "    return result_graph\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# evaluate_page_rank - Auxiliary function to calculate statistical data\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def evaluate_page_rank(topic, o_labels, sol_labels, **kwargs):\n",
    "    results = {}\n",
    "\n",
    "    results['accuracy'] = accuracy_score(sol_labels, o_labels)\n",
    "    results['precision-micro'] = precision_score(sol_labels, o_labels, average='micro', zero_division=1)\n",
    "    results['precision-macro'] = precision_score(sol_labels, o_labels, average='macro', zero_division=1)\n",
    "    results['recall-micro'] =  recall_score(sol_labels, o_labels, average='micro')\n",
    "    results['recall-macro'] =  recall_score(sol_labels, o_labels, average='macro')\n",
    "    results['f-beta-micro'] = fbeta_score(sol_labels, o_labels, average='micro', beta=0.5)\n",
    "    results['f-beta-macro'] = fbeta_score(sol_labels, o_labels, average='macro', beta=0.5)\n",
    "    results['MAP'] = average_precision_score(sol_labels, o_labels)\n",
    "\n",
    "    if 'curves' in kwargs and kwargs['curves']:\n",
    "        precision, recall, _ = precision_recall_curve(sol_labels, o_labels)\n",
    "        PrecisionRecallDisplay(precision=precision, recall=recall).plot()\n",
    "        plt.title('Precision Recall curve for Ranked topic {}'.format(topic))\n",
    "        plt.show()\n",
    "\n",
    "    return results\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "# display_results_page_rank - Auxiliary function to display calculated statistical data\n",
    "# -------------------------------------------------------------------------------------------------\n",
    "def display_results_page_rank(q, results_page_rank):\n",
    "    print(\"\\nPage Rank Search:\")\n",
    "    for theta in results_page_rank:\n",
    "        result_str= ''\n",
    "        for m in results_page_rank[theta]:\n",
    "            result_str += '{} = {}, '.format(m, round(results_page_rank[theta][m],4)) \n",
    "        print(\"For theta={}: {}\".format(theta, result_str[:-2]))\n",
    "\n",
    "    return\n",
    "\n",
    "# -------------------------------------------------------------------------------------------------------\n",
    "# evaluation - Function that fully evaluates our IR model, providing full statiscal analysis for several\n",
    "# theta values across multiple topics\n",
    "#\n",
    "# Input: Q_test - The set of topics we will evaluate the perform of our IR model on\n",
    "#        R_test - The topic labels we are looking for\n",
    "#        D_test - Our test set in collection form\n",
    "#        **kwargs - Optional parameters with the following functionality (default values prefixed by *)\n",
    "#               theta_range [list of floats or None | *[100,200,300,400,500]] - List of theta values our model will test\n",
    "#               sim_method [*cosine | eucledian | manhattan] - Sim method our page rank graph will use\n",
    "#\n",
    "# Behaviour: The function provides full statistics for every topic in Q_test, using R_test and D_test.\n",
    "# For each theta in theta_range it will use undirected_page_rank() to rank the top p documents.\n",
    "#\n",
    "# Output: Full statistical analysis for the provided input args\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "def evaluation_gr(Q_test, R_test, D_test, **kwargs):\n",
    "\n",
    "    results_page_rank = {}\n",
    "    theta_range = [0.20, 0.25, 0.30, 0.35, 0.40] if 'theta_range' not in kwargs else kwargs['theta_range']\n",
    "    sim_method = 'cosine' if 'sim_method' not in kwargs else kwargs['sim_method']\n",
    "\n",
    "    for q in Q_test:\n",
    "        r_labels = find_R_test_labels(R_test[q])\n",
    "\n",
    "        for theta in theta_range:\n",
    "            page_rank_docs = undirected_page_rank(q, D_test, 100, sim_method, theta, **kwargs)\n",
    "            ranked_labels = find_ranked_query_labels(page_rank_docs, r_labels)\n",
    "\n",
    "            results_page_rank[theta] = evaluate_page_rank(q, ranked_labels[0][:, 1],ranked_labels[1][:, 1], **kwargs)\n",
    "            \n",
    "        display_results_page_rank(q, results_page_rank)\n",
    "        \n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The previous section composes the enterity of our code that's not directly tied to the main functions for each topic, including every auxiliary function."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.1 - Clustering approach: organizing collections"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " **clustering(D, kwargs)** Applies Clustering, a unsupervised learning technique to classify \n",
    " sets of documents instead of individual documents\n",
    "\n",
    " **Input:** (D, kwargs)\n",
    " \n",
    "       D - set of documents or topics to be clustered\n",
    "       \n",
    "       kwargs - Optional parameters with the following functionality (default \n",
    "       values prefixed by *)\n",
    "           clusters [*range(2,20) | list of ints > 0]: List with the number of clusters to attempt in \n",
    "           the clustering algorithms\n",
    "           distance [*euclidian | manhattan]: Distance measurement used \n",
    "           top_cluster_words [*5 | int]: Number of top words that represent each cluster \n",
    "\n",
    " **Behaviour:** Starts by obtaining the processed collection of documents. Then vectorizes\n",
    " them throught the function tfidf_process and obtains the vectorizer itself, the document\n",
    " keys and the document vector. Afterwards, we obtain the r_set entries relevant to the doc keys,\n",
    " and deal with the kwargs information. These arguments are sent to the clustering training functions\n",
    " which return the information regarding the best clustering they found. We compare KM and AC through\n",
    " their best rand score, and then process the information to output.\n",
    "\n",
    " **Output:** A list of cluster results. These results consist in a pair per cluster, with the cluster \n",
    " centroid and the set of document/topic ids which comprise it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Document set results\n",
      "[[['helped', 'growing', 'try', 'damage', 'hawaii'], [4949, 4951, 4952, 4955, 4956, 4957, 4958, 4961, 4962, 4963]], [['high', 'day', '10', 'say', 'market'], [4948, 4950, 4953, 4954, 4959, 4960]]]\n",
      "[[['stock', 'world', 'added', 'plan', 'called'], [4952, 4955, 4959, 4962]], [['maker', 'corp', 'administration', 'friday', 'thursday'], [4948, 4960]], [['high', 'day', '10', 'say', 'market'], [4949, 4951, 4956, 4961, 4963]], [['market', 'high', 'maker', 'rose', 'world'], [4957, 4958]], [['year', 'usa', '10', 'high', 'time'], [4950, 4953, 4954]]]\n",
      "\n",
      "Topic set results\n",
      "[[['weight', 'loss', 'drug', 'harmful', 'east'], [101, 102, 103, 104, 105, 107, 108, 109, 110, 111, 113, 114, 115, 116, 117, 118, 119]], [['government', 'supported', 'voucher', 'death', 'mining'], [106, 112, 120]]]\n",
      "[[['boat', 'convict', 'ferry', 'offender', 'repeat'], [108, 111, 113, 114, 115, 116, 117, 118]], [['case', 'custody', 'kidnapped', 'rescue', 'child'], [101, 102, 103, 105, 119]], [['government', 'supported', 'voucher', 'death', 'mining'], [104, 109]], [['east', 'middle', 'terrorism', 'britain', 'great'], [107, 110]], [['loss', 'harmful', 'drug', 'weight', 'effect'], [106, 112, 120]]]\n"
     ]
    }
   ],
   "source": [
    "def clustering(D, **kwargs):\n",
    "    global tfidf_vec_info\n",
    "    global labels_pred\n",
    "\n",
    "    mode = 'docs' if 'mode' not in kwargs else kwargs['mode']\n",
    "\n",
    "    doc_keys = None\n",
    "    doc_vectors = None\n",
    "    y = []\n",
    "\n",
    "    if mode == 'docs':\n",
    "        tfidf_vec_info = tfidf_process(D, remove_stopwords='english', **kwargs)\n",
    "        doc_keys = tfidf_vec_info[1]\n",
    "        doc_vectors = tfidf_vec_info[2]\n",
    "\n",
    "        r_set = get_R_set('material/',index='doc_id')[0][1]\n",
    "    \n",
    "        for i in range(len(doc_keys)):\n",
    "            y.append([])\n",
    "            if doc_keys[i] in r_set:\n",
    "                for r in r_set[doc_keys[i]]:\n",
    "                    y[i].append('{}_{}'.format(r, r_set[doc_keys[i]][r]))\n",
    "        y = np.array(y, dtype=object)\n",
    "\n",
    "    elif mode == 'topics':\n",
    "        tfidf_vec_info = tfidf_process(D, min_df = 1, max_df=1.0, **kwargs)\n",
    "        doc_keys = tfidf_vec_info[1]\n",
    "        doc_vectors = tfidf_vec_info[2]\n",
    "\n",
    "        r_set = get_R_set('material/')[0][1]\n",
    "\n",
    "        for i in range(len(doc_keys)):\n",
    "            y.append([])\n",
    "            if doc_keys[i] in r_set:\n",
    "                for r in r_set[doc_keys[i]]:\n",
    "                    y[i].append('{}_{}'.format(r, r_set[doc_keys[i]][r]))\n",
    "        y = np.array(y, dtype=object)\n",
    "\n",
    "    clustering_methods = [trainKmeans, trainAgglomerative] if 'methods' not in kwargs else kwargs['methods']\n",
    "\n",
    "    clusters = list(range(2,100)) if 'clusters' not in kwargs else kwargs['clusters']\n",
    "    distances = ['euclidean'] if 'distance' not in kwargs else kwargs['distance']\n",
    "    top_cluster_words = 5 if 'top_cluster_words' not in kwargs else kwargs['top_cluster_words']\n",
    "    \n",
    "    # [Object, Labels, Score, n_clusters]\n",
    "    best_clusters = [None, None, 0, 0]\n",
    "\n",
    "    for method in clustering_methods:\n",
    "        for dist in distances:\n",
    "            clustering = method(doc_vectors, y, clusters, dist)\n",
    "\n",
    "            if clustering[2] > best_clusters[2]:\n",
    "                best_clusters = clustering\n",
    "\n",
    "    result = []\n",
    "    doc_labels = best_clusters[1]\n",
    "    labels_pred = best_clusters[1]\n",
    "\n",
    "    for i in range(best_clusters[3]):\n",
    "        result.append([None,[]])\n",
    "\n",
    "    centroids = None\n",
    "\n",
    "    if type(best_clusters[0]) == KMeans:\n",
    "        centroids = np.array(best_clusters[0].cluster_centers_)\n",
    "\n",
    "        for i in range(len(doc_keys)):\n",
    "            centroid = doc_labels[i]\n",
    "            result[centroid][1].append(doc_keys[i])\n",
    "\n",
    "    else:\n",
    "        docs_per_centroid = {}\n",
    "        for i in range(len(doc_keys)):\n",
    "            centroid = doc_labels[i]\n",
    "            result[centroid][1].append(doc_keys[i])\n",
    "\n",
    "            if centroid not in docs_per_centroid:\n",
    "                docs_per_centroid[centroid] = [doc_vectors[i].toarray()[0]]\n",
    "            else:\n",
    "                docs_per_centroid[centroid].append(doc_vectors[i].toarray()[0])\n",
    "\n",
    "        centroids = []\n",
    "        for centroid in docs_per_centroid:\n",
    "            value = np.mean(docs_per_centroid[centroid], axis=0)\n",
    "            centroids.append(value)\n",
    "\n",
    "    for i in range(len(centroids)):\n",
    "        aux_centroid = centroids[i][centroids[i] != 0.]\n",
    "        sorted_args = np.argsort(aux_centroid)\n",
    "\n",
    "        inverse_transformed = tfidf_vec_info[0].inverse_transform(centroids[i])[0]\n",
    "        n_args = len(inverse_transformed)\n",
    "        entry_range = top_cluster_words if n_args > top_cluster_words else n_args\n",
    "\n",
    "        centroid_result = []\n",
    "        for j in range(entry_range):\n",
    "            centroid_result.append(inverse_transformed[sorted_args[j]])\n",
    "\n",
    "        entry = centroid_result\n",
    "        result[i][0] = entry\n",
    "\n",
    "    return result\n",
    "\n",
    "\n",
    "# Some simple example executions\n",
    "# Change this to given rcv1 directory. Small sample size\n",
    "D_set = get_files_from_directory('../rcv1_test/19960821/', None)[1]\n",
    "D_set = process_collection(D_set, False)\n",
    "\n",
    "print('\\nDocument set results')\n",
    "print(clustering(D_set, method = [trainKmeans], clusters=list(range(2,5)), top_cluster_words=5))\n",
    "print(clustering(D_set, method = [trainAgglomerative], clusters=list(range(5,7)), top_cluster_words=5))\n",
    "\n",
    "material_dic = 'material/'\n",
    "topics = get_topics(material_dic)\n",
    "Q_set = get_topic_subset(list(range(101,121)))\n",
    "\n",
    "print('\\nTopic set results')\n",
    "print(clustering(Q_set, method = [trainKmeans], clusters=list(range(2,5)), top_cluster_words=5, mode='topics'))\n",
    "print(clustering(Q_set, method = [trainAgglomerative], clusters=list(range(5,7)), top_cluster_words=5, mode='topics'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**interpret(cluster, D, kwargs)** Outputs the representation of a cluster cointaining a median (centroid) and a medoid \n",
    "\n",
    " **Input:** (cluster, D, kwargs)\n",
    " \n",
    "            cluster - A document/topic cluster\n",
    "            D - Set of documents or topics in cluster\n",
    "            kwargs - Optional parameters with the following functionality (default \n",
    "            values prefixed by *)\n",
    "\n",
    " **Behaviour:** Creates a TF-IDF Vectorizer to process pairwise distances between each document in\n",
    " a cluster to calculate its medoid, and uses previous information to represent the centroid.\n",
    "\n",
    " **Output:** A list with centroid representation in top cluster words, and a medoid represented by\n",
    " a document or topic id "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Intrepet clustering results\n",
      "\n",
      "Example set\n",
      "[['mad', 'jakob', 'cow', 'creutzfeldt', 'britain'], 132]\n",
      "[['china', 'missile', 'pakistan', 'plant', 'nuclear'], 121]\n",
      "[['labor', 'law', 'television', 'child'], 128]\n",
      "[['anti', 'drug', 'rejection', 'organ', 'pig'], 133]\n",
      "\n",
      "Example set\n",
      "[['drug', 'anti', 'rejection', 'problem', 'planning'], 139]\n",
      "[['labor', 'law', 'television', 'child'], 128]\n",
      "[['china', 'missile', 'pakistan', 'plant', 'nuclear'], 121]\n",
      "[['fire', 'friendly', 'sea', 'turtle', 'death'], 132]\n",
      "[['britain', 'great', 'statistic', 'abuse', 'substance'], 134]\n",
      "[['cow', 'creutzfeldt', 'jakob', 'mad', 'parkinson'], 122]\n"
     ]
    }
   ],
   "source": [
    "def interpret(cluster, D, **kwargs):\n",
    "    documents = {}\n",
    "    for doc_id in cluster[1]:\n",
    "        documents[doc_id] = D[doc_id]\n",
    "\n",
    "    doc_vectors = tfidf_process(documents, min_df = 1, max_df=1.0, **kwargs)[2]\n",
    "    distance_matrix = pairwise_distances(doc_vectors)\n",
    "\n",
    "    centroid = cluster[0]\n",
    "    medoid_index = np.argmin(distance_matrix.sum(axis=0))\n",
    "\n",
    "    return [centroid, cluster[1][medoid_index]]\n",
    "\n",
    "# Some simple example executions\n",
    "Q_set = get_topic_subset(list(range(121,141)))\n",
    "\n",
    "clustering_set_1 = clustering(Q_set, method = [trainKmeans], clusters=list(range(2,5)), top_cluster_words=5, mode='topics')\n",
    "clustering_set_2 = clustering(Q_set, method = [trainAgglomerative], clusters=list(range(5,7)), top_cluster_words=5, mode='topics')\n",
    "sets = [clustering_set_1, clustering_set_2]\n",
    "\n",
    "print('\\nIntrepet clustering results')\n",
    "for clust_set in sets:\n",
    "    print('\\nExample set')\n",
    "    for cluster in clust_set:\n",
    "        print(interpret(cluster, Q_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**evaluate(D, kwargs)** Serves as the main class for the clustering process and outputs its results \n",
    "\n",
    " **Input:** (D, kwargs)\n",
    " \n",
    "         D - Document or topic collection to use\n",
    "         kwargs - Optional parameters with the following functionality (default \n",
    "         values prefixed by *)\n",
    "            mode [*docs | topics]: Chooses if we are running a topic or document collection \n",
    "\n",
    "**Behaviour:** It encapsulates the entirity of the clustering process and prints its output results\n",
    "ordered by the result of each individual cluster.\n",
    "\n",
    "**Output:** None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example Execution with documents\n",
      "\n",
      "The clustering solution has k = 2 clusters\n",
      "\n",
      "Cluster 1:\n",
      "Centroid has top words ['high', 'day', 'market', 'american', 'way']\n",
      "Medoid is document with id 4953\n",
      "Cluster is composed by 7 documents\n",
      "documents in cluster -> [4948, 4950, 4953, 4954, 4955, 4959, 4960]\n",
      "\n",
      "Cluster 2:\n",
      "Centroid has top words ['helped', 'known', 'try', 'growing', 'damage']\n",
      "Medoid is document with id 4957\n",
      "Cluster is composed by 9 documents\n",
      "documents in cluster -> [4949, 4951, 4952, 4956, 4957, 4958, 4961, 4962, 4963]\n",
      "Example Execution with documents\n",
      "\n",
      "The clustering solution has k = 3 clusters\n",
      "\n",
      "Cluster 1:\n",
      "Centroid has top words ['market', 'high', 'maker', 'rose', 'beat']\n",
      "Medoid is document with id 4957\n",
      "Cluster is composed by 7 documents\n",
      "documents in cluster -> [4949, 4951, 4956, 4957, 4958, 4961, 4963]\n",
      "\n",
      "Cluster 2:\n",
      "Centroid has top words ['service', 'order', 'begin', 'want', 'number']\n",
      "Medoid is document with id 4953\n",
      "Cluster is composed by 3 documents\n",
      "documents in cluster -> [4950, 4953, 4954]\n",
      "\n",
      "Cluster 3:\n",
      "Centroid has top words ['high', 'day', '10', 'say', 'market']\n",
      "Medoid is document with id 4948\n",
      "Cluster is composed by 6 documents\n",
      "documents in cluster -> [4948, 4952, 4955, 4959, 4960, 4962]\n",
      "\n",
      "External evaluation for clustering solution:\n",
      "rands_score = 0.1598360655737705\n",
      "v_score = 0.5133393276359093\n",
      "complete_score = 0.6829630204518538\n",
      "fowlkes_score = 0.39339789623472154\n",
      "homogenity_score = 0.411209525392666\n",
      "mutual_score = 0.7125718078632992\n",
      "\n",
      "Final Mean supervised score = 0.4788862738587034\n",
      "\n",
      "Example Execution with topics\n",
      "\n",
      "The clustering solution has k = 4 clusters\n",
      "\n",
      "Cluster 1:\n",
      "Centroid has top words ['payment', 'balance', 'civil', 'accident', 'institution']\n",
      "Medoid is topic with id 145\n",
      "Cluster is composed by 6 topics\n",
      "topics in cluster -> [145, 146, 147, 149, 150, 155]\n",
      "\n",
      "Cluster 2:\n",
      "Centroid has top words ['art', 'culture', 'entertainment', 'judicial', 'legal']\n",
      "Medoid is topic with id 141\n",
      "Cluster is composed by 8 topics\n",
      "topics in cluster -> [141, 142, 143, 144, 148, 152, 154, 158]\n",
      "\n",
      "Cluster 3:\n",
      "Centroid has top words ['bond', 'marketing', 'science', 'technology', 'plan']\n",
      "Medoid is topic with id 157\n",
      "Cluster is composed by 2 topics\n",
      "topics in cluster -> [157, 159]\n",
      "\n",
      "Cluster 4:\n",
      "Centroid has top words ['market', 'contract', 'defence', 'politics', 'competition']\n",
      "Medoid is topic with id 160\n",
      "Cluster is composed by 4 topics\n",
      "topics in cluster -> [151, 153, 156, 160]\n"
     ]
    }
   ],
   "source": [
    "def evaluate(D, **kwargs):\n",
    "    global tfidf_vec_info\n",
    "    global labels_pred\n",
    "\n",
    "    mode = 'docs' if 'mode' not in kwargs else kwargs['mode']\n",
    "    doc_dic = D\n",
    "\n",
    "    if mode == 'docs':\n",
    "        doc_dic = process_collection(D, False, **kwargs)\n",
    "    elif mode == 'topics':\n",
    "        doc_dic = get_topic_subset(D)\n",
    "\n",
    "    clusters = clustering(doc_dic, **kwargs)\n",
    "    cluster_info = []\n",
    "\n",
    "    for cluster in clusters:\n",
    "        cluster_info.append(interpret(cluster, doc_dic, **kwargs))\n",
    "\n",
    "    n_clusters = len(clusters)\n",
    "    name = 'document' if mode == 'docs' else 'topic'\n",
    "\n",
    "    print(\"The clustering solution has k = {} clusters\".format(n_clusters))\n",
    "    for i in range(n_clusters):\n",
    "        print(\"\\nCluster {}:\".format(i+1))\n",
    "        print(\"Centroid has top words {}\".format(cluster_info[i][0]))\n",
    "        print(\"Medoid is {} with id {}\".format(name,cluster_info[i][1]))\n",
    "        print(\"Cluster is composed by {} {}s\".format(len(clusters[i][1]), name))\n",
    "        print(\"{}s in cluster -> {}\".format(name, clusters[i][1]))\n",
    "\n",
    "    if mode == 'docs' and 'external' in kwargs and kwargs['external']:\n",
    "        print('\\nExternal evaluation for clustering solution:')\n",
    "        doc_keys = tfidf_vec_info[1]\n",
    "        category_ids = get_category_ids(doc_keys)\n",
    "        final_score = get_clustering_score(None, category_ids, labels_pred, 'supervised')\n",
    "        print('\\nFinal Mean supervised score = {}'.format(final_score))\n",
    "    \n",
    "    return\n",
    "\n",
    "# Some simple example executions\n",
    "D_set = get_files_from_directory('../rcv1_test/19960821/', None)[1]\n",
    "Q_set = list(range(141,161))\n",
    "\n",
    "print('Example Execution with documents\\n')\n",
    "evaluate(D_set, clusters=list(range(2,3)))\n",
    "\n",
    "print('Example Execution with documents\\n')\n",
    "evaluate(D_set, clusters=list(range(3,5)), external=True)\n",
    "\n",
    "print('\\nExample Execution with topics\\n')\n",
    "evaluate(Q_set, clusters=list(range(4,6)), mode='topics')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.2 -  Supervised approach: incorporating relevance feedback"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**training(q, d_train, r_train, kwargs)** - Trains a classification model for a given topic\n",
    "\n",
    "**Input:** (q, d_train, r_train, kwargs)\n",
    "\n",
    "         q - topic document\n",
    "         d_train - training collection\n",
    "         r_train - judgements\n",
    "         kwargs - Optional parameters with the following functionality (default values prefixed by *)\n",
    "            classifier [*multinomialnb | kneighbors | randomforest | mlp] - Classifier model used for training\n",
    "    \n",
    "**Behaviour:** Learns a classification model to predict the relevance of documents on the topic q using the documents in the d_train collection, and their respective judgements r_train, that have been labeled for topic q. The vectors created from he subset of d_train relevant for topic q are stored in list for use in the classification function.\n",
    "\n",
    "**Output:** q-conditional classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training with q = 120\n",
      "MultinomialNB()\n",
      "\n",
      "Training with q = 140\n",
      "KNeighborsClassifier()\n",
      "\n",
      "Training with q = 160\n",
      "RandomForestClassifier()\n",
      "\n",
      "Training with q = 180\n",
      "MLPClassifier()\n"
     ]
    }
   ],
   "source": [
    "def training(q, d_train, r_train, **kwargs):\n",
    "    global topic_vectorizers\n",
    "\n",
    "    classifiers = {'multinomialnb': MultinomialNB(), 'kneighbors': KNeighborsClassifier(), 'randomforest': RandomForestClassifier(), 'mlp': MLPClassifier()}\n",
    "    classifier = classifiers['multinomialnb'] if 'classifier' not in kwargs else classifiers[kwargs['classifier']]\n",
    "\n",
    "    r_labels = find_R_test_labels(r_train[q])\n",
    "\n",
    "    subset_dtrain = {}\n",
    "    for doc in r_labels:\n",
    "        subset_dtrain[doc] = d_train[doc]\n",
    "\n",
    "    vec_results = create_vectorizer(subset_dtrain, 'tf-idf', **kwargs)\n",
    "    topic_vectorizers[q] = vec_results[0]\n",
    "    d_train_vec = vec_results[1]\n",
    "    \n",
    "    r_labels = list(r_labels.values())\n",
    "    \n",
    "    classifier.fit(X=d_train_vec, y=r_labels)\n",
    "\n",
    "    return classifier\n",
    "\n",
    "\n",
    "# Some simple example executions\n",
    "\n",
    "# Explicit ways of getting documents directly from rcv1\n",
    "#D_train = get_files_from_directory('../rcv1/', None, set='train')[1]\n",
    "#D_train = process_collection(D_train, None)\n",
    "\n",
    "# Example execution with results saved in hard drive\n",
    "d_train = read_from_file('collections_processed/Dtrain_judged_collection_processed')\n",
    "\n",
    "r_set = get_R_set(material_dic)[0]\n",
    "r_train = r_set[1]\n",
    "\n",
    "Q_Test = [120,140,160,180]\n",
    "classifiers = ['multinomialnb', 'kneighbors', 'randomforest', 'mlp']\n",
    "example_models = []\n",
    "\n",
    "for i in range(len(Q_Test)):\n",
    "    print('\\nTraining with q = {}'.format(Q_Test[i]))\n",
    "    example_models.append(training(Q_Test[i], d_train, r_train, classifier=classifiers[i]))\n",
    "    print(example_models[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**classify(d,q,M,kwargs)** - Calculates the probability of a document being relevant for a given topic\n",
    " \n",
    "**Input:** (d,q,M,kwargs)\n",
    "            \n",
    "           d - document\n",
    "           q - topic\n",
    "           M - classification model\n",
    "\n",
    "**Behaviour:** classifies the probability of document d to be relevant for topic q given M, using a vectorizer created during the training function\n",
    "\n",
    "**Output:** probabilistic classification output on the relevance of document d to the topic q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Example execution for topic 120\n",
      "Probability of being relevant for topic is 12.121054825591909%\n",
      "\n",
      "Example execution for topic 140\n",
      "Probability of being relevant for topic is 20.0%\n",
      "\n",
      "Example execution for topic 160\n",
      "Probability of being relevant for topic is 9.0%\n",
      "\n",
      "Example execution for topic 180\n",
      "Probability of being relevant for topic is 0.9919804654565064%\n"
     ]
    }
   ],
   "source": [
    "def classify(d, q, M, **kwargs):\n",
    "    vec = None\n",
    "    vectorizers = topic_vectorizers[q]\n",
    "\n",
    "    if type(vectorizers) != list:\n",
    "        vec = vectorizers.transform(d)\n",
    "    else:\n",
    "        vec = vectorizers[1].transform(vectorizers[0].transform(d))\n",
    "\n",
    "    return M.predict_proba(vec)[0][1]\n",
    "\n",
    "\n",
    "# Some simple example executions\n",
    "example_docs = get_files_from_directory('../rcv1_test/19960821/', None)[1][0:4]\n",
    "example_docs = list(process_collection(example_docs, False).values())\n",
    "\n",
    "for i in range(len(Q_Test)):\n",
    "    print('\\nExample execution for topic {}'.format(Q_Test[i]))\n",
    "    print('Probability of being relevant for topic is {}%'.format(classify([example_docs[i]], Q_Test[i], example_models[i])*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**evaluate(q_test, d_test, r_test, kwargs)** - Evaluates accuracy, precision and recall of the IR system using relevance feedback\n",
    " \n",
    "**Input:**  (q_test, d_test, r_test, kwargs)\n",
    "\n",
    "            q_test - subset of topics\n",
    "            d_test - testing document collection\n",
    "            r_test - judgements\n",
    "            kwargs - Optional parameters with the following functionality (default values prefixed by *)\n",
    "               ranking [*False | True] - Use of page ranking in evaluation\n",
    "               p [*5 | int] - Number of pages displayed in page ranking\n",
    " \n",
    "**Behaviour:** evaluates the behavior of the IR system in the presence of relevance feedback. Training and testing functions are called for each topic in Qtest for a more comprehensive assessment. Prints performance statistics regarding the underlying classification system and the behavior of the aided IR system\n",
    "\n",
    "**Output:** Returns total accuracy of the IR system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for example execution of evaluate\n",
      "\n",
      "Result for search on Topic 121\n",
      "accuracy = 0.8593, precision-micro = 0.8593, precision-macro = 0.9296, recall-micro = 0.8593, recall-macro = 0.5, f-beta-micro = 0.8593, f-beta-macro = 0.4421\n",
      "\n",
      "Result for search on Topic 122\n",
      "accuracy = 0.8702, precision-micro = 0.8702, precision-macro = 0.9351, recall-micro = 0.8702, recall-macro = 0.5, f-beta-micro = 0.8702, f-beta-macro = 0.4467\n",
      "\n",
      "Result for search on Topic 123\n",
      "accuracy = 0.9503, precision-micro = 0.9503, precision-macro = 0.9751, recall-micro = 0.9503, recall-macro = 0.5, f-beta-micro = 0.9503, f-beta-macro = 0.4799\n",
      "\n",
      "Result for search on Topic 124\n",
      "accuracy = 0.868, precision-micro = 0.868, precision-macro = 0.934, recall-micro = 0.868, recall-macro = 0.5, f-beta-micro = 0.868, f-beta-macro = 0.4458\n",
      "\n",
      "Result for search on Topic 125\n",
      "accuracy = 0.7592, precision-micro = 0.7592, precision-macro = 0.8794, recall-micro = 0.7592, recall-macro = 0.5038, f-beta-micro = 0.7592, f-beta-macro = 0.417\n",
      "\n",
      "Mean result for precision = 86.1401812835411%\n"
     ]
    }
   ],
   "source": [
    "def evaluate(q_test, d_test, r_test, **kwargs):\n",
    "    ranking = False if 'ranking' not in kwargs else kwargs['ranking']\n",
    "    p = 5 if 'top_p' not in kwargs else kwargs['top_p']\n",
    "\n",
    "    total_accuracy = 0\n",
    "\n",
    "    for q in q_test:\n",
    "\n",
    "        sol_labels = []\n",
    "        o_labels_training = []\n",
    "        trained_probs = {}\n",
    "\n",
    "        trained_classifier = training(q, d_train, r_train, **kwargs)\n",
    "\n",
    "        judged_docs = []\n",
    "        for doc_id in r_test[q]:\n",
    "            judged_docs.append(doc_id)\n",
    "        \n",
    "        for doc_id in judged_docs:\n",
    "\n",
    "            trained_prob = classify([d_test[doc_id]], q, trained_classifier)\n",
    "            trained_probs[doc_id] = trained_prob\n",
    "            sol_labels.append(r_test[q][doc_id])\n",
    "            o_labels_training.append(1 if trained_prob > 0.5 else 0)\n",
    "\n",
    "        results = {}\n",
    "        \n",
    "        results['accuracy'] = accuracy_score(sol_labels, o_labels_training)\n",
    "        results['precision-micro'] = precision_score(sol_labels, o_labels_training, average='micro', zero_division=1)\n",
    "        results['precision-macro'] = precision_score(sol_labels, o_labels_training, average='macro', zero_division=1)\n",
    "        results['recall-micro'] =  recall_score(sol_labels, o_labels_training, average='micro')\n",
    "        results['recall-macro'] =  recall_score(sol_labels, o_labels_training, average='macro')\n",
    "        results['f-beta-micro'] = fbeta_score(sol_labels, o_labels_training, average='micro', beta=0.5)\n",
    "        results['f-beta-macro'] = fbeta_score(sol_labels, o_labels_training, average='macro', beta=0.5)\n",
    "\n",
    "        if ranking:\n",
    "            sort_probs = sorted(trained_probs, key = trained_probs.get, reverse=True)\n",
    "\n",
    "            ranked_result = []\n",
    "            result_range = range(p) if p <= len(sort_probs) else range(len(sort_probs))\n",
    "            for i in result_range:\n",
    "                doc_id = sort_probs[i]\n",
    "                ranked_result.append((doc_id, round(trained_probs[doc_id], 4)))\n",
    "\n",
    "            results['ranking'] = ranked_result\n",
    "\n",
    "        display_results(q, results)\n",
    "\n",
    "        total_accuracy += results['accuracy']\n",
    "\n",
    "    return total_accuracy / len(q_test)\n",
    "\n",
    "# Some simple example executions\n",
    "\n",
    "# Explicit ways of getting documents directly from rcv1\n",
    "# D_test = get_files_from_directory('../rcv1/', None, set='test')[0]\n",
    "# D_test = process_collection(D_test, None)\n",
    "\n",
    "# Example execution with results saved in hard drive\n",
    "D_test = read_from_file('collections_processed/Dtest_judged_collection_processed')\n",
    "Q_Test_eva = list(range(121,126))    \n",
    "r_test = r_set[0]\n",
    "\n",
    "print('Results for example execution of evaluate\\n')\n",
    "print('Mean result for accuracy = {}%'.format(evaluate(Q_Test_eva, D_test, r_test)*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.3 - Graph ranking approach: document centrality"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**build_graph(D, sim, theta, kwargs)** - Builds a document graph from document collection D using\n",
    "the similarity measure in sim agains theta threshold\n",
    "\n",
    "**Input:** (D, sim, theta, kwargs)\n",
    "\n",
    "            D - The document collection to build our graph with\n",
    "            sim - [*cosine | euclidean | manhattan] : The similarity measure used\n",
    "            theta - The similarity threshold \n",
    "\n",
    "**Behaviour:** This function starts by creating the necessary structures for each of the give graph entries, and then proceeds to calculate the necessary pairwise similarity measures. It does so by treating each individual document as the query document and comparing it to all the rest.\n",
    "\n",
    "**Output:** A dictionary representing the weighted undirect graph that connect all documents on the basis of the given similarity measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example execution of build graph\n",
      "\n",
      "For cosine similarity measure, we have:\n",
      "{4948: {}, 4949: {4957: 0.3388390448714529, 4958: 0.3388390448714529}, 4950: {4953: 0.3972091985292888, 4954: 0.3972091985292888}, 4951: {4961: 0.3083232047998879}, 4952: {}, 4953: {4950: 0.3972091985292888, 4954: 1.0, 4957: 0.3649077964672879, 4958: 0.3649077964672879}, 4954: {4950: 0.3972091985292888, 4953: 1.0, 4957: 0.3649077964672879, 4958: 0.3649077964672879}, 4955: {}, 4956: {}, 4957: {4949: 0.3388390448714529, 4953: 0.3649077964672879, 4954: 0.3649077964672879, 4958: 0.9999999999999996, 4961: 0.3198734348087981}, 4958: {4949: 0.3388390448714529, 4953: 0.3649077964672879, 4954: 0.3649077964672879, 4957: 0.9999999999999996, 4961: 0.3198734348087981}, 4959: {}, 4960: {}, 4961: {4951: 0.3083232047998879, 4957: 0.3198734348087981, 4958: 0.3198734348087981}, 4962: {}, 4963: {}}\n",
      "\n",
      "For euclidean similarity measure, we have:\n",
      "{4948: {4949: 0.7129455178270572, 4950: 0.7311760540835823, 4951: 0.7170515389116583, 4952: 0.730247030583577, 4953: 0.7404524310271149, 4954: 0.7404524310271149, 4955: 0.7286840900761609, 4956: 0.7263941610554607, 4957: 0.7255031120335983, 4958: 0.7255031120335983, 4959: 0.7548543787085075, 4960: 0.7524046431271271, 4961: 0.748713347372307, 4962: 0.710871123078811, 4963: 0.710726130489337}, 4949: {4948: 0.7129455178270572, 4950: 0.7240721650095189, 4951: 0.7804210438307182, 4952: 0.7282129000544333, 4953: 0.7305833521704477, 4954: 0.7305833521704477, 4955: 0.728784160639407, 4956: 0.8234112249827682, 4957: 0.8696237717535263, 4958: 0.8696237717535263, 4959: 0.7293473988559255, 4960: 0.7271943331887374, 4961: 0.8440319020996907, 4962: 0.7446466576981059, 4963: 0.814723343390458}, 4950: {4948: 0.7311760540835823, 4949: 0.7240721650095189, 4951: 0.730344868315953, 4952: 0.7325753384792433, 4953: 0.910755272184698, 4954: 0.910755272184698, 4955: 0.7628548397106139, 4956: 0.7243061350858259, 4957: 0.8303298720076552, 4958: 0.8303298720076552, 4959: 0.7606647114534923, 4960: 0.7192629065869809, 4961: 0.7252486098861955, 4962: 0.7136471322319696, 4963: 0.7078066058260026}, 4951: {4948: 0.7170515389116583, 4949: 0.7804210438307182, 4950: 0.730344868315953, 4952: 0.7394649480981441, 4953: 0.7433280376994965, 4954: 0.7433280376994965, 4955: 0.7258501588511749, 4956: 0.7535449026218748, 4957: 0.766573969148628, 4958: 0.766573969148628, 4959: 0.736089610232635, 4960: 0.7613224081167828, 4961: 0.8502240767231458, 4962: 0.7094316622679667, 4963: 0.7319218910239116}, 4952: {4948: 0.730247030583577, 4949: 0.7282129000544333, 4950: 0.7325753384792433, 4951: 0.7394649480981441, 4953: 0.7165655153696957, 4954: 0.7165655153696957, 4955: 0.7375059134124206, 4956: 0.812298377944997, 4957: 0.7294495019434252, 4958: 0.7294495019434252, 4959: 0.7325895257784335, 4960: 0.7123610941244275, 4961: 0.7186013281367288, 4962: 0.7091323549159555, 4963: 0.7267854895454833}, 4953: {4948: 0.7404524310271149, 4949: 0.7305833521704477, 4950: 0.910755272184698, 4951: 0.7433280376994965, 4952: 0.7165655153696957, 4955: 0.7305933802740191, 4956: 0.7300932428783982, 4957: 0.8872920932775652, 4958: 0.8872920932775652, 4959: 0.7504644102444948, 4960: 0.7241366208751372, 4961: 0.7386256778708357, 4962: 0.7166832864461373, 4963: 0.7228663072453823}, 4954: {4948: 0.7404524310271149, 4949: 0.7305833521704477, 4950: 0.910755272184698, 4951: 0.7433280376994965, 4952: 0.7165655153696957, 4955: 0.7305933802740191, 4956: 0.7300932428783982, 4957: 0.8872920932775652, 4958: 0.8872920932775652, 4959: 0.7504644102444948, 4960: 0.7241366208751372, 4961: 0.7386256778708357, 4962: 0.7166832864461373, 4963: 0.7228663072453823}, 4955: {4948: 0.7286840900761609, 4949: 0.728784160639407, 4950: 0.7628548397106139, 4951: 0.7258501588511749, 4952: 0.7375059134124206, 4953: 0.7305933802740191, 4954: 0.7305933802740191, 4956: 0.7681115460007957, 4957: 0.7170969778685732, 4958: 0.7170969778685732, 4959: 0.7373204025440052, 4960: 0.7149633244333778, 4961: 0.7236608746519837, 4962: 0.7071067811865475, 4963: 0.7071067811865474}, 4956: {4948: 0.7263941610554607, 4949: 0.8234112249827682, 4950: 0.7243061350858259, 4951: 0.7535449026218748, 4952: 0.812298377944997, 4953: 0.7300932428783982, 4954: 0.7300932428783982, 4955: 0.7681115460007957, 4957: 0.7742375311350659, 4958: 0.7742375311350659, 4959: 0.7122049464191729, 4960: 0.7285323824026658, 4961: 0.797753699861148, 4962: 0.7352725155366865, 4963: 0.7994660257176531}, 4957: {4948: 0.7255031120335983, 4949: 0.8696237717535263, 4950: 0.8303298720076552, 4951: 0.766573969148628, 4952: 0.7294495019434252, 4953: 0.8872920932775652, 4954: 0.8872920932775652, 4955: 0.7170969778685732, 4956: 0.7742375311350659, 4959: 0.727617742553134, 4960: 0.7227886314114199, 4961: 0.857413136294539, 4962: 0.7702362308265606, 4963: 0.8194520928810833}, 4958: {4948: 0.7255031120335983, 4949: 0.8696237717535263, 4950: 0.8303298720076552, 4951: 0.766573969148628, 4952: 0.7294495019434252, 4953: 0.8872920932775652, 4954: 0.8872920932775652, 4955: 0.7170969778685732, 4956: 0.7742375311350659, 4959: 0.727617742553134, 4960: 0.7227886314114199, 4961: 0.857413136294539, 4962: 0.7702362308265606, 4963: 0.8194520928810833}, 4959: {4948: 0.7548543787085075, 4949: 0.7293473988559255, 4950: 0.7606647114534923, 4951: 0.736089610232635, 4952: 0.7325895257784335, 4953: 0.7504644102444948, 4954: 0.7504644102444948, 4955: 0.7373204025440052, 4956: 0.7122049464191729, 4957: 0.727617742553134, 4958: 0.727617742553134, 4960: 0.7148078293746785, 4961: 0.7227210184570801, 4962: 0.7071067811865476, 4963: 0.7071067811865475}, 4960: {4948: 0.7524046431271271, 4949: 0.7271943331887374, 4950: 0.7192629065869809, 4951: 0.7613224081167828, 4952: 0.7123610941244275, 4953: 0.7241366208751372, 4954: 0.7241366208751372, 4955: 0.7149633244333778, 4956: 0.7285323824026658, 4957: 0.7227886314114199, 4958: 0.7227886314114199, 4959: 0.7148078293746785, 4961: 0.7247402539000997, 4962: 0.7118613019473372, 4963: 0.7116778019319571}, 4961: {4948: 0.748713347372307, 4949: 0.8440319020996907, 4950: 0.7252486098861955, 4951: 0.8502240767231458, 4952: 0.7186013281367288, 4953: 0.7386256778708357, 4954: 0.7386256778708357, 4955: 0.7236608746519837, 4956: 0.797753699861148, 4957: 0.857413136294539, 4958: 0.857413136294539, 4959: 0.7227210184570801, 4960: 0.7247402539000997, 4962: 0.7239960493908899, 4963: 0.7496219127531991}, 4962: {4948: 0.710871123078811, 4949: 0.7446466576981059, 4950: 0.7136471322319696, 4951: 0.7094316622679667, 4952: 0.7091323549159555, 4953: 0.7166832864461373, 4954: 0.7166832864461373, 4955: 0.7071067811865475, 4956: 0.7352725155366865, 4957: 0.7702362308265606, 4958: 0.7702362308265606, 4959: 0.7071067811865476, 4960: 0.7118613019473372, 4961: 0.7239960493908899, 4963: 0.7303081454439863}, 4963: {4948: 0.710726130489337, 4949: 0.814723343390458, 4950: 0.7078066058260026, 4951: 0.7319218910239116, 4952: 0.7267854895454833, 4953: 0.7228663072453823, 4954: 0.7228663072453823, 4955: 0.7071067811865474, 4956: 0.7994660257176531, 4957: 0.8194520928810833, 4958: 0.8194520928810833, 4959: 0.7071067811865475, 4960: 0.7116778019319571, 4961: 0.7496219127531991, 4962: 0.7303081454439863}}\n"
     ]
    }
   ],
   "source": [
    "def build_graph(D, sim, theta, **kwargs):\n",
    "    doc_dic = process_collection(D, False, **kwargs)\n",
    "    \n",
    "    tfidf_vectorizer_info = tfidf_process(doc_dic, **kwargs)\n",
    "\n",
    "    vectorizer = tfidf_vectorizer_info[0]\n",
    "    doc_keys = tfidf_vectorizer_info[1]\n",
    "    doc_vectors = tfidf_vectorizer_info[2]\n",
    "\n",
    "    graph = {}\n",
    "    for doc in doc_dic:\n",
    "        graph[doc] = {}\n",
    "\n",
    "    sim_method = sim_method_helper(sim)\n",
    "\n",
    "    for doc in doc_dic:\n",
    "        similarity_dic = sim_method([doc_dic[doc]], vectorizer, doc_keys, doc_vectors, theta, **kwargs)\n",
    "\n",
    "        for simil_doc in similarity_dic:\n",
    "            if doc != simil_doc:\n",
    "                graph[doc][simil_doc] = similarity_dic[simil_doc]\n",
    "                graph[simil_doc][doc] = similarity_dic[simil_doc]\n",
    "\n",
    "    return graph\n",
    "\n",
    "# Some simple example executions\n",
    "\n",
    "# Explicit ways of getting documents directly from rcv1\n",
    "# D_test = get_files_from_directory('../rcv1/', None, set='test')[0]\n",
    "# D_test = process_collection(D_test, None)\n",
    "\n",
    "# Example execution with results saved in hard drive\n",
    "D_test = get_files_from_directory('../rcv1_test/19960821/', None)[1]\n",
    "example_methods = ['cosine', 'euclidean']\n",
    "\n",
    "print('Example execution of build graph')\n",
    "for method in example_methods:\n",
    "    print('\\nFor {} similarity measure, we have:'.format(method))\n",
    "    print(build_graph(D_test, method, 0.3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**undirected_page_rank(q, D, p, sim, theta, kwargs)** - This function applies a modified version of the PageRank algorithm for undirected graphs to the provided document collection, retriving the top p documents for topic q in regars to similarity measure sim and treshold theta. \n",
    "\n",
    "**Input:** (q, D, p, sim, theta, kwargs)\n",
    "        \n",
    "        q - A topic query in the form of topic identifier (int)\n",
    "        D - The document collection we built our graph with\n",
    "        p - The number of top documents to return\n",
    "        sim - [*cosine | eucledian | manhattan] : The similarity measure used\n",
    "        theta - The similarity threshold\n",
    "        kwargs - Optional parameters with the following functionality (default values prefixed by *)\n",
    "               sim_weight [*0.5 | float in [0.0, 1.0] ]: The weight given to the base similarity measure\n",
    "               over the PageRank results \n",
    " \n",
    "**Behaviour:** This function serves primarily as an encapsulation for the PageRank algorithm, and as such it starts by creating the necessary structures for it to run, namely the link_graph. Afterwards, it  takes the PageRank results present in PageRank and weights the final results in accordance to the  results from the similarity measure sim given similiraty weight sim_weight. In the end it selects the top p perfoming documents for query q and returns them in list form.\n",
    "\n",
    "**Output:** A list of ordered top-documents with their corresponding score in the form (d, score), ordered in descending order of score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example execution of undirected_page_rank with uniform priors\n",
      "\n",
      "For topic q = 131 and top p = 5, we have:\n",
      "[(4950, 1.8255146691488435), (4953, 1.3714312455161008), (4954, 1.3714312455161008), (4961, 0.26397098767838123), (4949, 0.26397098767838106)]\n",
      "\n",
      "For topic q = 132 and top p = 5, we have:\n",
      "[(4953, 1.8228756555322954), (4954, 1.8228756555322954), (4961, 0.31101776349538646), (4949, 0.31101776349538635), (4950, 0.31101776349538635)]\n",
      "\n",
      "For topic q = 133 and top p = 5, we have:\n",
      "[(4950, 2.0679784744191054), (4957, 1.1355546454973477), (4958, 1.1355546454973477), (4953, 0.5007915700489632), (4954, 0.5007915700489632)]\n",
      "\n",
      "For topic q = 134 and top p = 5, we have:\n",
      "[(4961, 1.8785192153908028), (4950, 1.7651042280216762), (4949, 0.3111697540419658), (4951, 0.3111697540419658), (4953, 0.3111697540419658)]\n",
      "Example execution of undirected_page_rank with non-uniform priors (len of document)\n",
      "\n",
      "For topic q = 131 and top p = 5, we have:\n",
      "[(4950, 1.8059475015241249), (4953, 1.417340867124166), (4954, 1.417340867124166), (4957, 0.3317988923558549), (4958, 0.3317988923558549)]\n",
      "\n",
      "For topic q = 132 and top p = 5, we have:\n",
      "[(4953, 1.8687852771403606), (4954, 1.8687852771403606), (4957, 0.37884566817286014), (4958, 0.37884566817286014), (4949, 0.301688786879451)]\n",
      "\n",
      "For topic q = 133 and top p = 5, we have:\n",
      "[(4950, 2.048411306794387), (4957, 1.2033825501748217), (4958, 1.2033825501748217), (4953, 0.5467011916570286), (4954, 0.5467011916570286)]\n",
      "\n",
      "For topic q = 134 and top p = 5, we have:\n",
      "[(4961, 1.8251445783370825), (4950, 1.7455370603969576), (4957, 0.3789976587194396), (4958, 0.3789976587194396), (4953, 0.357079375650031)]\n"
     ]
    }
   ],
   "source": [
    "def undirected_page_rank(q, D, p, sim, theta, **kwargs):\n",
    "    link_graph = build_graph(D, sim, theta, **kwargs)\n",
    "    ranked_graph = page_rank(link_graph, q, D, **kwargs)\n",
    "\n",
    "    query = topics[q]\n",
    "    sim_method = sim_method_helper(sim)\n",
    "\n",
    "    tdidf_info = tfidf_process(process_collection(D, False), **kwargs)\n",
    "    vectorizer = tdidf_info[0]\n",
    "    doc_keys = tdidf_info[1]\n",
    "    doc_vectors = tdidf_info[2]\n",
    "\n",
    "    sim_dic = sim_method([query], vectorizer, doc_keys, doc_vectors, 0, **kwargs)\n",
    "\n",
    "    sim_weight = 0.5 if 'sim_weight' not in kwargs else kwargs['sim_weight']\n",
    "    pr_weight = 1 - sim_weight\n",
    "\n",
    "    ranked_graph = normalize_dic(ranked_graph, norm_method='zscore')\n",
    "    sim_dic = normalize_dic(sim_dic, norm_method='zscore')\n",
    "    \n",
    "    for doc in sim_dic:\n",
    "        sim_dic[doc] = sim_weight * sim_dic[doc] + pr_weight * ranked_graph[doc]\n",
    "\n",
    "    # Retrieve top p documents\n",
    "    sorted_dic = sorted(sim_dic, key = sim_dic.get, reverse=True)\n",
    "\n",
    "    result = []\n",
    "    result_range = range(p) if p <= len(sorted_dic) else range(len(sorted_dic))\n",
    "    for i in result_range:\n",
    "        doc = sorted_dic[i]\n",
    "        result += [(doc, sim_dic[doc]),]\n",
    "\n",
    "    return result\n",
    "\n",
    "# Some simple example executions\n",
    "\n",
    "q_test = list(range(131,135))\n",
    "p = 5\n",
    "print('Example execution of undirected_page_rank with uniform priors')\n",
    "for q in q_test:\n",
    "    print('\\nFor topic q = {} and top p = {}, we have:'.format(q, p))\n",
    "    print(undirected_page_rank(q, D_test, p, 'cosine', 0.3))\n",
    "    \n",
    "print('\\nExample execution of undirected_page_rank with non-uniform priors (len of document)')\n",
    "for q in q_test:\n",
    "    print('\\nFor topic q = {} and top p = {}, we have:'.format(q, p))\n",
    "    print(undirected_page_rank(q, D_test, p, 'cosine', 0.3, prior='non-uniform', prior_method='len'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}