IR-A1-K173654.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and declerations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import os\n",
    "import string\n",
    "import re\n",
    "import pprint\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "import numpy as np\n",
    "import copy\n",
    "from nltk.stem import PorterStemmer\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Structure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PostingList(object):\n",
    "    def __init__(self):\n",
    "        self.total_count = 0\n",
    "        self.token = ''\n",
    "        self.occurrance = {\n",
    "#             'doc_id':0 = 'positions' : [],\n",
    "#             \n",
    "        }\n",
    "    def toJSON(self):\n",
    "        return json.dumps(self, default=lambda o: o.__dict__, \n",
    "            sort_keys=True, indent=4)\n",
    "         \n",
    "    def __repr__(self):\n",
    "        \n",
    "        return f'total_cnt : {self.total_count} docs : [{self.occurrance.keys()}]'\n",
    "    def __len__(self):\n",
    "        return len(self.occurrance)\n",
    "    def addOccurrance(self, doc_id, position):\n",
    "        self.total_count += 1\n",
    "#         print(position)\n",
    "        if doc_id not in self.occurrance.keys():\n",
    "            self.occurrance[doc_id] = []\n",
    "        self.occurrance[doc_id].append(position)\n",
    "#         self.occurrance[doc_id]['position'].append(pos)\n",
    "    \n",
    "class InvertedIndex(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.index = {}\n",
    "        self.docs = {}\n",
    "\n",
    "    def get_term_postings(self, term):\n",
    "        if term in self.index.keys():\n",
    "            return self.index[term]\n",
    "        else:\n",
    "            raise ValueError(f'{term} is not present in Index. Plese Try Again')\n",
    "            return PostingList()\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.index.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Punctuation\n",
    "def remove_punctuation(word):\n",
    "    return word.translate(word.maketrans('','',string.punctuation))\n",
    "\n",
    "# Clean Query Term\n",
    "def clean_word(word):\n",
    "    # Case Folding\n",
    "    word = word.lower()\n",
    "     # Filter non-ASCII characters\n",
    "    word = ''.join(filter(lambda x: x in printable, word))\n",
    "#     print(word)\n",
    "    # Remove Punctuations\n",
    "    if word != '(' and word != ')':\n",
    "        word = remove_punctuation(word)\n",
    "#     print(word)\n",
    "    if re.match('\\d+[A-Za-z]+',word):\n",
    "        word = re.split('\\d+',word)[1]\n",
    "    if re.match('[A-Za-z]+\\d+',word):\n",
    "        word = re.split('\\d+',word)[0]\n",
    "#     print(word)\n",
    "    word = ps.stem(word)\n",
    "#     print(word)\n",
    "    return word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Term - Document Indexing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "doc_contents = []\n",
    "inverted_index = InvertedIndex()\n",
    "printable = set(string.printable) \n",
    "# Printable characters are\n",
    "# 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n",
    "# !\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~ \\t\\n\\r\\x0b\\x0c\n",
    "\n",
    "\n",
    "ps = PorterStemmer()\n",
    "\n",
    "stop_words = set()\n",
    "with open('Stopword-List.txt', 'r') as stop_word_file:\n",
    "    lines = stop_word_file.readlines()\n",
    "    for line in lines:\n",
    "        stop_words.add(line.split('\\n')[0])\n",
    "    stop_words.remove('')\n",
    "\n",
    "    \n",
    "    \n",
    "for file_number in range(0, 56):\n",
    "    with open(f'data/Trump Speechs/speech_{file_number}.txt', 'r') as file1:\n",
    "        lines = file1.readlines()\n",
    "#         print(f'File Number : speech_{file_number}.txt' )\n",
    "#         print(lines[0])\n",
    "        position = {'doc':file_number,'row':0, 'col':0, 'token_no':0}\n",
    "        \n",
    "#         {\n",
    "#             'total_count' : 0,\n",
    "#             'postings' : {\n",
    "#                 'count':0,\n",
    "#                 'doc_id':0,\n",
    "#                 'positions':[]\n",
    "#             }\n",
    "#         }\n",
    "        \n",
    "        for line_no,line in enumerate(lines):\n",
    "            # Skip Heading Line\n",
    "            if line_no == 0:\n",
    "                continue\n",
    "            doc_set = set()\n",
    "            # split words at . , whitespace ? ! : ;\n",
    "            position['row'] = line_no \n",
    "            position['col'] = 0\n",
    "            for word in re.split('[.\\s,?!:;-]', line):\n",
    "                position['col'] += len(word) + 1\n",
    "                position['token_no'] += 1\n",
    "                # Case Folding\n",
    "                word = word.lower()\n",
    "                \n",
    "                # Filter non-ASCII characters\n",
    "                word = ''.join(filter(lambda x: x in printable, word))\n",
    "                \n",
    "                # Remove Punctuations\n",
    "                word = remove_punctuation(word)\n",
    "                \n",
    "                if re.match('\\d+[A-Za-z]+',word):\n",
    "                    word = re.split('\\d+',word)[1]\n",
    "                if re.match('[A-Za-z]+\\d+',word):\n",
    "                    word = re.split('\\d+',word)[0]\n",
    "                \n",
    "                if len(word) == 0 or len(word) == 1 or word == '' or word == ' ':\n",
    "                    continue\n",
    "                if word in stop_words:\n",
    "                    continue\n",
    "                \n",
    "                \n",
    "                \n",
    "                word = ps.stem(word)\n",
    "                    \n",
    "                vocab.add(word)\n",
    "                \n",
    "                doc_set.add(word)\n",
    "                \n",
    "                if word in inverted_index.index.keys():\n",
    "                    \n",
    "                    inverted_index.index[word].addOccurrance(file_number, copy.deepcopy(position)) \n",
    "                else:\n",
    "                    plist = PostingList()\n",
    "                    inverted_index.index[word] = plist\n",
    "                    inverted_index.index[word].addOccurrance(file_number, copy.deepcopy(position))\n",
    "                    \n",
    "        inverted_index.docs[file_number] = doc_set\n",
    "        doc_contents.append(doc_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Vocabulary Size \n",
      "4781\n",
      "Total Number of Documents \n",
      "56\n"
     ]
    }
   ],
   "source": [
    "print('Total Vocabulary Size ')\n",
    "print(len(inverted_index.index.keys()))\n",
    "print('Total Number of Documents ')\n",
    "print(len(inverted_index.docs))\n",
    "# print('Occurances of query: hammer')\n",
    "# inverted_index.index[ps.stem('hammer')].occurrance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To save Inverted Index in File\n",
    "with open('pickled/inverted_index.p', 'wb') as index_file:\n",
    "    pickle.dump(inverted_index, index_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Load Inverted Index from File\n",
    "with open('pickled/inverted_index.p', 'rb') as index_file:\n",
    "    inverted_index = pickle.load(index_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersect_posting(p1, p2):\n",
    "    if len(p1) == 0 or len(p2) == 0:\n",
    "        return PostingList()\n",
    "\n",
    "    if isinstance(p1, set) and isinstance(p2, set):\n",
    "        return p1.intersection(p2)\n",
    "    elif isinstance(p1, set):\n",
    "        return p1.intersection(p2.occurrance.keys())\n",
    "    elif isinstance(p2, set):\n",
    "        return p2.intersection(p1.occurrance.keys())\n",
    "    pn = PostingList()\n",
    "    # pn.token = f'{p1.token} & {p2.token}'\n",
    "    for pn_keys in (p1.occurrance.keys() & p2.occurrance.keys()) :\n",
    "        pn.addOccurrance(pn_keys, p1.occurrance[pn_keys])\n",
    "        pn.addOccurrance(pn_keys, p2.occurrance[pn_keys])\n",
    "    return pn\n",
    "\n",
    "\n",
    "def union_posting(p1, p2):\n",
    "    if len(p1) == 0:\n",
    "        return p2\n",
    "    elif len(p2) == 0:\n",
    "        return p1\n",
    "    if isinstance(p1, set) and isinstance(p2, set):\n",
    "        return p1.union(p2)\n",
    "    elif isinstance(p1, set):\n",
    "        return p1.union(p2.occurrance.keys())\n",
    "    elif isinstance(p2, set):\n",
    "        return p2.union(p1.occurance.keys())\n",
    "    \n",
    "    pn = PostingList()\n",
    "    # pn.token = f'{p1.token} | {p2.token}'\n",
    "    for pn1_keys in p1.occurrance.keys() :\n",
    "        pn.addOccurrance(pn1_keys, p1.occurrance[pn1_keys])\n",
    "    for pn2_keys in p2.occurrance.keys() :\n",
    "        pn.addOccurrance(pn2_keys, p2.occurrance[pn2_keys])\n",
    "    \n",
    "    return pn\n",
    "\n",
    "def inverse_posting(inverted_index,p):\n",
    "    print(p)\n",
    "    if isinstance(p, set) :\n",
    "        print('Returning ')\n",
    "        print(set(inverted_index.docs).difference(p))\n",
    "        return set(inverted_index.docs).difference(p)\n",
    "    else:\n",
    "        print(set(inverted_index.docs).difference(set(p.occurrance.keys())))\n",
    "        return set(inverted_index.docs).difference(set(p.occurrance.keys()))\n",
    "    return inverted_index.docs.keys() - p.occurrance.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "def build_index():\n",
    "    path_to_data = os.path.dirname(__file__) + '../../data/'\n",
    "    print(os.path.dirname(__file__))\n",
    "    print(path_to_data)\n",
    "    vocab = set()\n",
    "    doc_contents = []\n",
    "    inverted_index = InvertedIndex()\n",
    "    printable = set(string.printable) \n",
    "    # Printable characters are\n",
    "    # 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n",
    "    # !\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~ \\t\\n\\r\\x0b\\x0c\n",
    "\n",
    "\n",
    "    ps = PorterStemmer()\n",
    "    stop_words = set()\n",
    "    with open(path_to_data+'Stopword-List.txt', 'r') as stop_word_file:\n",
    "        lines = stop_word_file.readlines()\n",
    "        for line in lines:\n",
    "            stop_words.add(line.split('\\n')[0])\n",
    "        stop_words.remove('')\n",
    "    print(stop_words)\n",
    "\n",
    "    for file_number in range(0, 56):\n",
    "        with open(path_to_data + f'Trump Speechs/speech_{file_number}.txt', 'r') as file1:\n",
    "            lines = file1.readlines()\n",
    "            print(f'File Number : speech_{file_number}.txt' )\n",
    "            print(lines[0])\n",
    "            position = {'doc':file_number,'row':0, 'col':0, 'token_no':0}\n",
    "\n",
    "            for line_no,line in enumerate(lines):\n",
    "                doc_set = set()\n",
    "                # split words at . , whitespace ? ! : ;\n",
    "                position['row'] = line_no \n",
    "                position['col'] = 0\n",
    "                for word in re.split('[.\\s,?!:;-]', line):\n",
    "                    position['col'] += len(word) + 1\n",
    "                    position['token_no'] += 1\n",
    "                    # Case Folding\n",
    "                    word = word.lower()\n",
    "                    \n",
    "                    # Filter non-ASCII characters\n",
    "                    word = ''.join(filter(lambda x: x in printable, word))\n",
    "                    \n",
    "                    # Remove Punctuations\n",
    "                    word = remove_punctuation(word)\n",
    "                    \n",
    "                    if re.match('\\d+[A-Za-z]+',word):\n",
    "                        word = re.split('\\d+',word)[1]\n",
    "                    if re.match('[A-Za-z]+\\d+',word):\n",
    "                        word = re.split('\\d+',word)[0]\n",
    "                    \n",
    "                    if len(word) == 0 or len(word) == 1 or word == '' or word == ' ':\n",
    "                        continue\n",
    "                    if word in stop_words:\n",
    "                        continue\n",
    "\n",
    "                    word = ps.stem(word)\n",
    "                        \n",
    "                    vocab.add(word)\n",
    "                    \n",
    "                    doc_set.add(word)\n",
    "                    \n",
    "                    if word in inverted_index.index.keys():\n",
    "                        \n",
    "                        inverted_index.index[word].addOccurrance(file_number, copy.deepcopy(position)) \n",
    "                    else:\n",
    "                        plist = PostingList()\n",
    "                        inverted_index.index[word] = plist\n",
    "                        inverted_index.index[word].addOccurrance(file_number, copy.deepcopy(position))\n",
    "                        \n",
    "            inverted_index.docs[file_number] = doc_set\n",
    "            doc_contents.append(doc_set)\n",
    "    ii = InvertedIndexModel()\n",
    "    ii.status = True\n",
    "    ii.data = inverted_index\n",
    "    ii.save()\n",
    "    return True\n",
    "\n",
    "\n",
    "boperators = ['and', 'or']\n",
    "uoperators = ['not']\n",
    "\n",
    "# Token types\n",
    "#\n",
    "# EOF (end-of-file) token is used to indicate that\n",
    "# there is no more input left for lexical analysis\n",
    "LPAREN, RPAREN, EOF, TERM, AND, OR, NOT = (\n",
    "    '(', ')', 'EOF', 'TERM', 'AND','OR', 'NOT'\n",
    ")\n",
    "\n",
    "\n",
    "class Token(object):\n",
    "    def __init__(self, type, value):\n",
    "        self.type = type\n",
    "        self.value = value\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "\n",
    "    def __str__(self):\n",
    "        \"\"\"String representation of the class instance.\n",
    "\n",
    "        Examples:\n",
    "            Token(TERM, Hello)\n",
    "            Token(AND, '&')\n",
    "            Token(NOT, '!')\n",
    "        \"\"\"\n",
    "        return 'Token({type}, {value})'.format(\n",
    "            type=self.type,\n",
    "            value=repr(self.value)\n",
    "        )\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.__str__()\n",
    "\n",
    "\n",
    "class Lexer(object):\n",
    "    def __init__(self, text):\n",
    "        # client string input, e.g. \"hello | world & (why | are | you)\"\n",
    "        self.text = text\n",
    "        # self.pos is an index into self.text\n",
    "        self.pos = 0\n",
    "        self.current_char = self.text[self.pos]\n",
    "        \n",
    "\n",
    "    def error(self):\n",
    "        raise Exception('Invalid character')\n",
    "\n",
    "    def advance(self):\n",
    "        \"\"\"Advance the `pos` pointer and set the `current_char` variable.\"\"\"\n",
    "        self.pos += 1\n",
    "        if self.pos > len(self.text) - 1:\n",
    "            self.current_char = None  # Indicates end of input\n",
    "        else:\n",
    "            self.current_char = self.text[self.pos]\n",
    "\n",
    "    def skip_whitespace(self):\n",
    "        while self.current_char is not None and self.current_char.isspace():\n",
    "            self.advance()\n",
    "\n",
    "    def integer(self):\n",
    "\n",
    "        result = ''\n",
    "        while self.current_char is not None and self.current_char.isdigit():\n",
    "            result += self.current_char\n",
    "            self.advance()\n",
    "        return int(result)\n",
    "    def word(self):\n",
    "        result = ''\n",
    "        # while self.current_char is not None and (self.current_char.isalpha() or self.current_char == '_'):\n",
    "        while self.current_char is not None and (self.current_char in printable) and (self.current_char not in (' ', '|','&','!', '(', ')')):\n",
    "            result += self.current_char\n",
    "            self.advance()\n",
    "        return str(result)\n",
    "\n",
    "    def get_next_token(self):\n",
    "        \"\"\"Lexical analyzer (also known as scanner or tokenizer)\n",
    "\n",
    "        This method is responsible for breaking a sentence\n",
    "        apart into tokens. One token at a time.\n",
    "        \"\"\"\n",
    "        while self.current_char is not None:\n",
    "\n",
    "            if self.current_char.isspace():\n",
    "                self.skip_whitespace()\n",
    "                continue\n",
    "                        \n",
    "            if self.current_char == '&':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(AND, 'AND')\n",
    "            \n",
    "            if self.current_char == '|':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(OR, 'OR')\n",
    "            \n",
    "            if self.current_char == '!':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(NOT,'NOT')\n",
    "            \n",
    "\n",
    "            if self.current_char == '(':\n",
    "                self.advance()\n",
    "                return Token(LPAREN, '(')\n",
    "\n",
    "            if self.current_char == ')':\n",
    "                self.advance()\n",
    "                return Token(RPAREN, ')')\n",
    "            \n",
    "            if self.current_char.isalpha():      \n",
    "#                 print('Got token  ' + self.current_char)\n",
    "                return Token(TERM, self.word())\n",
    "            \n",
    "            \n",
    "\n",
    "            self.error()\n",
    "\n",
    "        return Token(EOF, None)\n",
    "\n",
    "class AST(object):\n",
    "    pass\n",
    "\n",
    "\n",
    "class BinOp(AST):\n",
    "    def __init__(self, left, op, right):\n",
    "        self.left = left\n",
    "        self.token = self.op = op\n",
    "        self.right = right\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "        self.value = ''\n",
    "        \n",
    "class Num(AST):\n",
    "    def __init__(self, token):\n",
    "        self.token = token\n",
    "        self.value = token.value\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "\n",
    "class Parser(object):\n",
    "    def __init__(self, lexer):\n",
    "        self.lexer = lexer\n",
    "        # set current token to the first token taken from the input\n",
    "        self.current_token = self.lexer.get_next_token()\n",
    "\n",
    "    def error(self):\n",
    "        raise Exception('Invalid syntax')\n",
    "\n",
    "    def eat(self, token_type):\n",
    "        # compare the current token type with the passed token\n",
    "        # type and if they match then \"eat\" the current token\n",
    "        # and assign the next token to the self.current_token,\n",
    "        # otherwise raise an exception.\n",
    "        if self.current_token.type == token_type:\n",
    "            self.current_token = self.lexer.get_next_token()\n",
    "        else:\n",
    "            self.error()\n",
    "\n",
    "    def factor(self):\n",
    "        \"\"\"factor : INTEGER | LPAREN expr RPAREN\"\"\"\n",
    "        token = self.current_token\n",
    "        \n",
    "        \n",
    "        if token.type == TERM:\n",
    "            self.eat(TERM)\n",
    "            return Num(token)\n",
    "        \n",
    "        elif token.type == NOT:\n",
    "            self.eat(NOT)\n",
    "            node = self.expr()\n",
    "            node.inverse = True\n",
    "            return node\n",
    "            \n",
    "        \n",
    "        \n",
    "        elif token.type == LPAREN:\n",
    "            self.eat(LPAREN)\n",
    "            node = self.expr()\n",
    "            self.eat(RPAREN)\n",
    "            return node\n",
    "\n",
    "    def term(self):\n",
    "        \n",
    "        node = self.factor()\n",
    "\n",
    "        while self.current_token.type in (AND,):\n",
    "            token = self.current_token\n",
    "         \n",
    "            if token.type == AND:\n",
    "                self.eat(AND)\n",
    "            \n",
    "            node = BinOp(left=node, op=token, right=self.factor())\n",
    "\n",
    "        return node\n",
    "\n",
    "    def expr(self):\n",
    "        node = self.term()\n",
    "\n",
    "        while self.current_token.type in (OR,):\n",
    "            token = self.current_token\n",
    "            if token.type == OR:\n",
    "                self.eat(OR)\n",
    "            \n",
    "            node = BinOp(left=node, op=token, right=self.term())\n",
    "\n",
    "        return node\n",
    "\n",
    "    def parse(self):\n",
    "        return self.expr()\n",
    "\n",
    "\n",
    "class NodeVisitor(object):\n",
    "    def visit(self, node):\n",
    "#         print('Checking Node Name')\n",
    "        \n",
    "        method_name = 'visit_' + type(node).__name__\n",
    "        visitor = getattr(self, method_name, self.generic_visit)\n",
    "        return visitor(node)\n",
    "\n",
    "    def generic_visit(self, node):\n",
    "        raise Exception('No visit_{} method'.format(type(node).__name__))\n",
    "\n",
    "\n",
    "class Interpreter(NodeVisitor):\n",
    "    def __init__(self, parser, index, ps):\n",
    "        self.parser = parser\n",
    "        self.index = index\n",
    "        self.ps = ps\n",
    "\n",
    "    def visit_BinOp(self, node):\n",
    "#         print('Bin OP : ' )\n",
    "#         print(node.token)\n",
    "#         print(node.value)\n",
    "#         print(node.row)\n",
    "#         print(node.inverse)\n",
    "        if node.op.type == AND:\n",
    "#             print('Node => ')\n",
    "#             print(node)\n",
    "            left = self.visit(node.left)   \n",
    "            right = self.visit(node.right)\n",
    "            \n",
    "#             term_index_left = vocab_list.index(ps.stem(left.value))\n",
    "#             term_row_left = term_doc_matrix_np[term_index_left]\n",
    "            \n",
    "#             term_index_right = vocab_list.index(ps.stem(right.value))\n",
    "#             term_row_right = term_doc_matrix_np[term_index_right]\n",
    "            \n",
    "            if left.inverse == True:\n",
    "                left.value = '!' + str(left.value)\n",
    "                term_row_left = self.index.index[self.ps.stem(left.row)]\n",
    "                left.inverse = False\n",
    "            \n",
    "            if right.inverse == True:\n",
    "                right.value = '!' + str(right.value)\n",
    "                term_row_right = self.index.index[self.ps.stem(right.row)]\n",
    "                right.inverse = False\n",
    "            \n",
    "            node.row = intersect_posting(left.row, right.row)\n",
    "            if node.inverse == True:\n",
    "                node.row = inverse_posting(self.index, node.row)\n",
    "                node.inverse = False\n",
    "            \n",
    "            return node\n",
    "        \n",
    "        elif node.op.type == OR:\n",
    "#             print('Node => ')\n",
    "#             print(node)\n",
    "            left = self.visit(node.left)   \n",
    "            right = self.visit(node.right)\n",
    "            \n",
    "#             term_index_left = vocab_list.index(ps.stem(left.value))\n",
    "#             term_row_left = term_doc_matrix_np[term_index_left]\n",
    "            \n",
    "#             term_index_right = vocab_list.index(ps.stem(right.value))\n",
    "#             term_row_right = term_doc_matrix_np[term_index_right]\n",
    "            \n",
    "            if left.inverse == True:\n",
    "                left.value = '!' + str(left.value)\n",
    "                term_row_left = self.index.index([self.ps.stem(left.row)])\n",
    "                left.inverse = False\n",
    "            \n",
    "            if right.inverse == True:\n",
    "                right.value = '!' + str(right.value)\n",
    "                term_row_right = self.index.index([self.ps.stem(right.row)])\n",
    "                right.inverse = False\n",
    "            \n",
    "\n",
    "            node.row = union_posting(left.row, right.row)\n",
    "            if node.inverse == True:\n",
    "                node.row = inverse_posting(self.index, node.row)\n",
    "                node.inverse = False\n",
    "            \n",
    "            return node\n",
    "        \n",
    "\n",
    "    def visit_Num(self, node):\n",
    "#         print('Num  : ' )\n",
    "#         print(node.token)\n",
    "#         print(node.value)\n",
    "#         print(node.row)\n",
    "#         print(node.inverse)\n",
    "        \n",
    "        node.value = node.value.split('_')[0]\n",
    "        if self.ps.stem(node.value) in self.index.index.keys():\n",
    "            \n",
    "            term_docs = self.index.index[self.ps.stem(node.value)]\n",
    "            \n",
    "        else:\n",
    "            term_docs = {}\n",
    "#             print('Term Row')\n",
    "#             print(term_docs)\n",
    "            \n",
    "        node.row = term_docs\n",
    "        if node.inverse == True:\n",
    "            node.row = inverse_posting(self.index, node.row)\n",
    "            node.inverse = False\n",
    "        \n",
    "        return node\n",
    "\n",
    "    def interpret(self):\n",
    "        tree = self.parser.parse()\n",
    "#         print(tree)\n",
    "        return self.visit(tree)\n",
    "\n",
    "def get_boolean_query(query):\n",
    "    text = str(query)\n",
    "    text = text.replace(' and ','&')\n",
    "    text = text.replace(' AND ','&')\n",
    "    text = text.replace(' or ','|')\n",
    "    text = text.replace(' OR ','|')\n",
    "    text = text.replace('NOT', '!')\n",
    "    text = text.replace('not ','!')\n",
    "    \n",
    "    print(text)\n",
    "#     Django Specific Code\n",
    "#     inverted_index_model_obj = InvertedIndexModel.objects.get()\n",
    "#     inverted_index = inverted_index_model_obj.data\n",
    "#     print('Inverted Index')\n",
    "    \n",
    "    ps = PorterStemmer()\n",
    "    lexer = Lexer(text)\n",
    "    parser = Parser(lexer)\n",
    "    interpreter = Interpreter(parser, inverted_index, ps)\n",
    "    result = interpreter.interpret()\n",
    "\n",
    "#     print(result.value)\n",
    "#     print(result.row)\n",
    "    return result.row\n",
    "\n",
    "def positional_intersect(p1, p2, k):\n",
    "    \n",
    "    ip = intersect_posting(p1, p2)\n",
    "    \n",
    "    lip = sorted(list(ip.occurrance))\n",
    "    npl = PostingList()\n",
    "    ans = []\n",
    "    \n",
    "    for doc in lip:\n",
    "#         print(type(p1))\n",
    "        positions1 = p1.occurrance[doc]\n",
    "        positions2 = p2.occurrance[doc]\n",
    "        index_p2 = 0\n",
    "        index_p1 = 0\n",
    "        for pos1 in positions1:\n",
    "            for pos2 in positions2:\n",
    "                if pos2['token_no'] -  pos1['token_no'] == k and pos2['token_no'] -  pos1['token_no'] > 0:\n",
    "                    ans.append({'doc':doc, 'pos1':  pos1, 'pos2':pos2})\n",
    "                    npl.addOccurrance(doc,pos1)\n",
    "                    npl.addOccurrance(doc,pos2)\n",
    "        \n",
    "        \n",
    "    return npl\n",
    "        \n",
    "\n",
    "def get_phrasal_query(query):\n",
    "    text = str(query)\n",
    "\n",
    "    try:\n",
    "        q1, q2 = text.split(' ')\n",
    "    except ValueError as e:\n",
    "        raise ValueError('Invalid Phrasal Query Syntax')\n",
    "    ps = PorterStemmer()\n",
    "    q1 = ps.stem(q1)\n",
    "    q2 = ps.stem(q2)\n",
    "#     Django Specific Code\n",
    "#     inverted_index_model_obj = InvertedIndexModel.objects.get()\n",
    "#     inverted_index = inverted_index_model_obj.data\n",
    "#     print('Inverted Index')\n",
    "    result = [] \n",
    "    p1 = inverted_index.get_term_postings(q1)\n",
    "    p2 = inverted_index.get_term_postings(q2)\n",
    "    \n",
    "    result = positional_intersect(p1, p2, 1)\n",
    "    \n",
    "    return result\n",
    "\n",
    "def get_proximity_query(query):\n",
    "    text = str(query)\n",
    "    try:\n",
    "        q1, q2, q3 = text.split(' ')\n",
    "    except ValueError as e:\n",
    "        raise ValueError('Invalid Proximity Query Syntax')\n",
    "    ps = PorterStemmer()\n",
    "    q1 = ps.stem(q1)\n",
    "    q2 = ps.stem(q2)\n",
    "    k = int(q3[1])+ 1\n",
    "#     Django Specific Code\n",
    "#     inverted_index_model_obj = InvertedIndexModel.objects.get()\n",
    "#     inverted_index = inverted_index_model_obj.data\n",
    "    \n",
    "    print('Inverted Index')\n",
    "    result = [] \n",
    "    \n",
    "    p1 = inverted_index.get_term_postings(q1)\n",
    "    p2 = inverted_index.get_term_postings(q2)\n",
    "    print(p1)\n",
    "    result = positional_intersect(p1, p2, k)\n",
    "    \n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Provided Queries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Boolean Queries '"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'running'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!hammer\n",
      "total_cnt : 29 docs : [dict_keys([16, 17, 18, 19, 20, 21, 24, 25, 27, 33, 34, 35, 36, 39, 40, 42, 43, 45, 46, 49, 50, 51, 53, 54])]\n",
      "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 22, 23, 26, 28, 29, 30, 31, 32, 37, 38, 41, 44, 47, 48, 52, 55}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'not hammer'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "actions&wanted\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'actions AND wanted'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "policies&western\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'policies AND western'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "united|plane\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'united OR plane'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amazing&playing&here\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'amazing AND playing AND here'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "higher&signals&enjoyed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'higher AND signals AND enjoyed'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "higher|signals|enjoyed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'higher OR signals OR enjoyed'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "forgiveness|developments|praised\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'forgiveness OR developments OR praised'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pakistan|afghanistan|aid\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'pakistan OR afghanistan OR aid'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "praised|( forgiveness&developments )\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'praised OR ( forgiveness AND developments )'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "officially|(warriors&cartoons)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'officially OR (warriors AND cartoons)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "businessperson&( nationwide|international )\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'businessperson AND ( nationwide OR international )'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outdated&( personnel|policies)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'outdated AND ( personnel OR policies)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outdated|( personnel&policies )\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'outdated OR ( personnel AND policies )'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "biggest&( near|box )\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'biggest AND ( near OR box )'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "box&( united|year )\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'box AND ( united OR year )'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "biggest&( plane|wanted|hour)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'biggest AND ( plane OR wanted OR hour)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "! (united&plane)\n",
      "total_cnt : 34 docs : [dict_keys([0, 1, 2, 32, 33, 34, 35, 36, 17, 19, 52, 24, 25, 26, 27, 29, 30])]\n",
      "{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 22, 23, 28, 31, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 53, 54, 55}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'NOT (united AND plane)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "! (higher|signals|enjoyed)\n",
      "total_cnt : 23 docs : [dict_keys([2, 7, 8, 12, 19, 20, 21, 23, 24, 30, 31, 39, 40, 43, 53, 0, 1, 22, 27, 37, 41, 44])]\n",
      "{3, 4, 5, 6, 9, 10, 11, 13, 14, 15, 16, 17, 18, 25, 26, 28, 29, 32, 33, 34, 35, 36, 38, 42, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'NOT (higher OR signals OR enjoyed)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "' phrasal queries '"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hillary Clinton\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Hillary Clinton'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "' positional queries'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after years /1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'after years /1'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "develop solutions /1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'develop solutions /1'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keep out /2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'keep out /2'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fails = []\n",
    "\n",
    "\"\"\" Boolean Queries \"\"\"\n",
    "q = 'running'\n",
    "a = {'0', '1', '10', '11', '12', '16', '17', '18', '19', '2', '20', '21', '22', '24', '25', '26', '27', '28', '3', '30', '32', '33', '34', '35', '36', '37', '39', '4', '40', '41', '44', '45', '46', '47', '5', '50', '51', '52', '53', '6', '8', '9'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "\n",
    "q = 'not hammer'\n",
    "a = {'31', '28', '37', '30', '7', '10', '14', '1', '6', '41', '15', '11', '29', '26', '52', '13', '32', '44', '4', '8', '22', '38', '48', '0', '47', '2', '23', '9', '3', '5', '12', '55'}\n",
    "r = get_boolean_query(q)\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'actions AND wanted'\n",
    "a = {'37', '3', '19', '1', '9', '40', '51', '16', '15', '12', '31', '41', '39', '0', '53', '26', '29', '17', '24', '54', '7', '2', '5', '28', '42'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'policies AND western'\n",
    "a ={'3', '2', '9'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'united OR plane'\n",
    "a = {'31', '28', '50', '46', '37', '30', '54', '10', '18', '7', '1', '17', '41', '49', '6', '34', '36', '11', '45', '29', '26', '52', '13', '21', '24', '16', '25', '32', '33', '4', '44', '22', '8', '19', '40', '20', '38', '48', '0', '47', '27', '51', '43', '2', '35', '39', '9', '3', '5', '12', '55'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'amazing AND playing AND here'\n",
    "a = {'51', '8', '0', '13', '39', '18', '19', '41', '20', '40', '16', '21', '50', '11', '27'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'higher AND signals AND enjoyed'\n",
    "a = {'8'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'higher OR signals OR enjoyed'\n",
    "a = {'8', '0', '39', '24', '41', '22', '44', '12', '23', '30', '31', '27', '20', '1', '40', '43', '19', '53', '7', '37', '2', '21'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'forgiveness OR developments OR praised'\n",
    "a = {'3', '8', '18', '20', '32', '43', '9', '17', '36', '44', '52', '42', '37', '38', '16', '31', '5', '2'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'pakistan OR afghanistan OR aid'\n",
    "a = {'29', '16', '4', '22', '37', '40', '42', '18', '1', '17', '41', '39', '9', '3'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'praised OR ( forgiveness AND developments )'\n",
    "a = {'36', '42', '44', '2'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'officially OR (warriors AND cartoons)'\n",
    "a = {'3', '0', '39', '18', '12', '48', '38', '5', '31', '10', '11', '47', '9', '17', '7', '37', '21', '25', '49'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'businessperson AND ( nationwide OR international )'\n",
    "a = set({})\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'outdated AND ( personnel OR policies)'\n",
    "a = {'2', '8', '5'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'outdated OR ( personnel AND policies )'\n",
    "a = {'42', '8', '18', '5', '2', '11'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'biggest AND ( near OR box )'\n",
    "a = {'18', '46', '54', '43', '4', '45', '50', '53', '47', '6', '51', '44'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'box AND ( united OR year )'\n",
    "a = {'18', '46', '4', '45', '50', '9', '47', '23', '54', '44', '25'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'biggest AND ( plane OR wanted OR hour)'\n",
    "a = {'50', '53', '46', '37', '30', '54', '42', '18', '7', '1', '49', '41', '6', '36', '45', '26', '52', '44', '16', '4', '8', '19', '40', '48', '0', '47', '51', '43', '2', '35', '39'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'NOT (united AND plane)'\n",
    "a = {'31', '28', '50', '53', '46', '37', '54', '42', '7', '10', '14', '18', '6', '49', '41', '15', '11', '45', '13', '21', '44', '16', '4', '8', '22', '40', '20', '38', '48', '47', '51', '43', '23', '39', '9', '3', '5', '12', '55'}\n",
    "r = get_boolean_query(q)\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'NOT (higher OR signals OR enjoyed)'\n",
    "a = {'3', '52', '16', '36', '10', '14', '34', '9', '17', '42', '55', '50', '49', '4', '18', '13', '29', '48', '28', '5', '38', '26', '35', '45', '11', '54', '51', '6', '47', '32', '46', '15', '33', '25'}\n",
    "r = get_boolean_query(q)\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "\n",
    "\"\"\" phrasal queries \"\"\"\n",
    "q = 'Hillary Clinton' \n",
    "a = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '14', '16', '17', '18', '19', '20', '21', '22', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36', '37', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48', '49', '50', '51', '52', '53', '54'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "\n",
    "\"\"\" positional queries\"\"\"\n",
    "q = 'after years /1'\n",
    "a = {'6', '7', '44'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'develop solutions /1'\n",
    "a = {'5', '32'}\n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})\n",
    "    \n",
    "q = 'keep out /2'\n",
    "a = {'20', '24', '39', '40', '51'} \n",
    "r = get_boolean_query(q).occurrance.keys()\n",
    "rs = set([str(x) for x in r])\n",
    "if (len(a.difference(rs)) == 0):\n",
    "    q\n",
    "else:\n",
    "    'Fail ' + q\n",
    "fails.append({q:[a.difference(rs), rs.difference(a)]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# If Result is not Empty, Some Queries are not returning correct docs\n",
    "list(filter(lambda k : 'FAILED : '+str(list(k.keys)[0]) if len(list(k.values())[0][0]) > 0 else None ,fails))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test to check least frequent terms\n",
    "# m = 100\n",
    "# val = []\n",
    "# for p in inverted_index.index:\n",
    "#     print(p)\n",
    "#     if len(p) < m:\n",
    "#         m = len(p)\n",
    "#         val = p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Some test queries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter Boolean Query: e.g outdated OR ( personnel AND policies ) : outdated OR ( personnel AND policies )\n",
      "outdated|( personnel&policies )\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "total_cnt : 7 docs : [dict_keys([2, 5, 8, 18, 11, 42])]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = input('Enter Boolean Query: e.g outdated OR ( personnel AND policies ) : ')\n",
    "get_boolean_query(q)\n",
    "# Output contains posting list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter phrasal Query: e.g Hillary ClintonHillary Clinton\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "total_cnt : 1116 docs : [dict_keys([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54])]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = input('Enter phrasal Query: e.g Hillary Clinton')\n",
    "get_phrasal_query(q)\n",
    "# Output contains posting list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter Proximity Query: e.g develop solutions /1 : develop solutions /1\n",
      "Inverted Index\n",
      "total_cnt : 33 docs : [dict_keys([2, 3, 5, 8, 9, 16, 17, 18, 20, 31, 32, 37, 38])]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "total_cnt : 4 docs : [dict_keys([5, 32])]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = input('Enter Proximity Query: e.g develop solutions /1 : ')\n",
    "get_proximity_query(q)\n",
    "# Output contains posting list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set(inverted_index.docs).symmetric_difference(set(inverted_index.get_term_postings('candidaci').occurrance.keys()))\n",
    "# set(inverted_index.get_term_postings('candidaci').occurrance.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set(inverted_index.get_term_postings('candidaci').occurrance.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References :\n",
    "\n",
    "http://www.pyregex.com/\n",
    "http://cs231n.github.io/python-numpy-tutorial/\n",
    "\n",
    "https://www.online-utility.org/text/analyzer.jsp\n",
    "\n",
    "https://stackoverflow.com/questions/2118261/parse-boolean-arithmetic-including-parentheses-with-regex\n",
    "\n",
    "https://regex101.com/r/M8z3U4/1\n",
    "\n",
    "https://iq.opengenus.org/porter-stemmer/\n",
    "\n",
    "https://unnikked.ga/how-to-build-a-boolean-expression-evaluator-518e9e068a65\n",
    "\n",
    "https://ruslanspivak.com/lsbasi-part7/\n",
    "\n",
    "https://github.com/gintas/django-picklefield"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit ('A1': pipenv)",
   "language": "python",
   "name": "python37464bita1pipenv857e3569e98b4a86911659abcc01ba46"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}