examples/notebooks/indexing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Indexing Demo.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TNdMHD8LS2Bm",
        "colab_type": "text"
      },
      "source": [
        "# PyTerrier Indexing Demo\n",
        "\n",
        "This notebook takes you through indexing using [PyTerrier](https://github.com/terrier-org/pyterrier)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pkMrNonZrpEg",
        "colab_type": "text"
      },
      "source": [
        "## Prerequisites\n",
        "\n",
        "You will need PyTerrier installed. PyTerrier also needs Java to be installed, and will find most installations."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JWLqWXBHeBRc",
        "colab_type": "code",
        "outputId": "04d8b4fa-7e60-4ccc-ad6b-3b95f0445437",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 765
        }
      },
      "source": [
        "!pip install python-terrier\n",
        "#!pip install --upgrade git+https://github.com/terrier-org/pyterrier.git#egg=python-terrier"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting python-terrier\n",
            "  Cloning https://github.com/terrier-org/pyterrier.git to /tmp/pip-install-1xbb7l2t/python-terrier\n",
            "  Running command git clone -q https://github.com/terrier-org/pyterrier.git /tmp/pip-install-1xbb7l2t/python-terrier\n",
            "Collecting pyjnius~=1.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/50/098cb5fb76fb7c7d99d403226a2a63dcbfb5c129b71b7d0f5200b05de1f0/pyjnius-1.3.0-cp36-cp36m-manylinux2010_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from python-terrier) (1.18.4)\n",
            "Requirement already satisfied, skipping upgrade: pandas in /usr/local/lib/python3.6/dist-packages (from python-terrier) (1.0.4)\n",
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Collecting pytrec_eval\n",
            "  Downloading https://files.pythonhosted.org/packages/36/0a/5809ba805e62c98f81e19d6007132712945c78e7612c11f61bac76a25ba3/pytrec_eval-0.4.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from python-terrier) (4.41.1)\n",
            "Collecting matchpy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/47/95/d265b944ce391bb2fa9982d7506bbb197bb55c5088ea74448a5ffcaeefab/matchpy-0.5.1-py3-none-any.whl (67kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.1MB/s \n",
            "\u001b[?25hCollecting deprecation\n",
            "  Downloading https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: cython in /usr/local/lib/python3.6/dist-packages (from pyjnius~=1.3.0->python-terrier) (0.29.19)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.7.0 in /usr/local/lib/python3.6/dist-packages (from pyjnius~=1.3.0->python-terrier) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas->python-terrier) (2.8.1)\n",
            "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->python-terrier) (2018.9)\n",
            "Collecting multiset<3.0,>=2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a8/12/813a649f5bc9801865dc6cda95b8f169f784d996322db192907ebe399064/multiset-2.1.1-py2.py3-none-any.whl\n",
            "Collecting hopcroftkarp<2.0,>=1.2\n",
            "  Downloading https://files.pythonhosted.org/packages/6b/56/7b03eba3c43008c490c9d52e69ea5334b65955f66836eb4f1962f3b0d421/hopcroftkarp-1.2.5.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from deprecation->python-terrier) (20.4)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->deprecation->python-terrier) (2.4.7)\n",
            "Building wheels for collected packages: python-terrier, wget, pytrec-eval, hopcroftkarp\n",
            "  Building wheel for python-terrier (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-terrier: filename=python_terrier-0.1.3-cp36-none-any.whl size=29566 sha256=6929651f224399255efa6ea2d368e9a434ef43362b829d8c019f228cf1f1d848\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-jg2en2vu/wheels/cc/bb/69/836d846a92c787b35ca6478119c0033762ab2b95d866eeb288\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp36-none-any.whl size=9682 sha256=b618609a13f95e8bf46899190330a98c380e95e999f4bbfabac784f9d9064535\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "  Building wheel for pytrec-eval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pytrec-eval: filename=pytrec_eval-0.4-cp36-cp36m-linux_x86_64.whl size=273904 sha256=c6cdd7a5a3fb03bc3e83782ffdbe0b89a297df7d251d92f4e3459f9857d5a6d3\n",
            "  Stored in directory: /root/.cache/pip/wheels/58/30/73/8858a1b6e5e2674e2ea85c9904949c06addcf6fd34d59b5ea6\n",
            "  Building wheel for hopcroftkarp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hopcroftkarp: filename=hopcroftkarp-1.2.5-py2.py3-none-any.whl size=18092 sha256=3422723d4dacff52d79490bf029f0f489472a2dd657a4e44868a9aef5f6733c3\n",
            "  Stored in directory: /root/.cache/pip/wheels/2b/e1/c9/1993c7f7f114b7d3fb2d3e895e02157a7ebf554861e9e54e01\n",
            "Successfully built python-terrier wget pytrec-eval hopcroftkarp\n",
            "Installing collected packages: pyjnius, wget, pytrec-eval, multiset, hopcroftkarp, matchpy, deprecation, python-terrier\n",
            "Successfully installed deprecation-2.1.0 hopcroftkarp-1.2.5 matchpy-0.5.1 multiset-2.1.1 pyjnius-1.3.0 python-terrier-0.1.3 pytrec-eval-0.4 wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ixzDjvtTOQbB",
        "colab_type": "text"
      },
      "source": [
        "## Init \n",
        "\n",
        "You must run `pt.init()` before other pyterrier functions and classes\n",
        "\n",
        "Optional Arguments:    \n",
        " - `version` - terrier IR version e.g. \"5.2\"    \n",
        " - `mem` - megabytes allocated to java e.g. \"4096\"      \n",
        " - `packages` - external java packages for Terrier to load e.g. [\"org.terrier:terrier.prf\"]\n",
        " - `logging` - logging level for Terrier. Defaults to \"WARN\", use \"INFO\" or \"DEBUG\" for more output.\n",
        "\n",
        "NB: PyTerrier needs Java 11 installed. If it cannot find your Java installation, you can set the `JAVA_HOME` environment variable."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k3ltUZ8PgWmz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "647fe3a9-6171-4ad0-8537-17c90aa03943"
      },
      "source": [
        "import pyterrier as pt\n",
        "if not pt.started():\n",
        "  pt.init()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "terrier-assemblies 5.2  jar-with-dependencies not found, downloading to /root/.pyterrier...\n",
            "Done\n",
            "terrier-python-helper 0.0.2  jar not found, downloading to /root/.pyterrier...\n",
            "Done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "By5UFYnRLgD0",
        "colab_type": "text"
      },
      "source": [
        "## TREC Indexing\n",
        "\n",
        "Here, we are going to make use of Pyterrier's dataset API. We will use the [vaswani_npl corpus](http://ir.dcs.gla.ac.uk/resources/test_collections/npl/), a very small information retrieval test collection. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRx5kIL9nmsB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c63da1f0-8d73-4c36-bc0e-ab8366779840"
      },
      "source": [
        "dataset = pt.datasets.get_dataset(\"vaswani\")\n",
        "\n",
        "print(\"Files in vaswani corpus: %s \" % dataset.get_corpus())"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files in vaswani corpus: ['/root/.pyterrier/corpora/vaswani/corpus/doc-text.trec'] \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXq3z6MdgWi4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "index_path = \"./index\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gjsYdSDvOTi_",
        "colab_type": "text"
      },
      "source": [
        "Create `pt.TRECCollectionIndexer` object    \n",
        "index_path argument specifies where to store the index"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Va70HfBN7s7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "c74112c1-acca-488c-bd86-07f278265507"
      },
      "source": [
        "!rm -rf ./index\n",
        "indexer = pt.TRECCollectionIndexer(index_path, blocks=True)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "IndexingType.CLASSIC\n",
            "IndexingType.CLASSIC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ak_VGdxfQm0p",
        "colab_type": "text"
      },
      "source": [
        "Index the files by calling the index method on the TRECCollectionIndexer object"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3C3NLUwQq0p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "indexref = indexer.index(dataset.get_corpus())\n",
        "\n",
        "# indexer method takes either a string or a list of strings with the files names\n",
        "# indexer.index([\"/vaswani_corpus/doc-text.trec\",])\n",
        "# indexer.index(\"/vaswani_corpus/doc-text.trec\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cYDrBpM6ofW-",
        "colab_type": "text"
      },
      "source": [
        "Lets see what we got from the indexer.\n",
        "\n",
        "IndexRef is a python object representing a Terrier [IndexRef](http://terrier.org/docs/current/javadoc/org/terrier/querying/IndexRef.html) object. You can think of this like a pointer, or a URI. In this case, it points to the location of the main index file."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hI9iNQdpoegm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a34fa4e1-a8de-4843-ab47-4db649c88f5c"
      },
      "source": [
        "indexref.toString()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'./index/data.properties'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hf7hX8xXpN6q",
        "colab_type": "text"
      },
      "source": [
        "We can use that to get more information about the index. For instance, to see the statistics of the index, lets use `index.getCollectionStatistics().toString()`. You can see that we have indexed 11429 documents, containing a total of 7756 unique words."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_wbIUBzeoqkZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "180e7910-274e-4b08-d974-da47033d4e87"
      },
      "source": [
        "index = pt.IndexFactory.of(indexref)\n",
        "print(index.getCollectionStatistics().toString())"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of documents: 11429\n",
            "Number of terms: 7756\n",
            "Number of fields: 0\n",
            "Field names: []\n",
            "Number of tokens: 271581\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yVjVoDYfTN2z",
        "colab_type": "text"
      },
      "source": [
        "To index TXT, PDF, Microsoft Word, etc files use pt.FilesIndexer instead of pt.TRECCollectionIndexer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wGV18JV1P129",
        "colab_type": "text"
      },
      "source": [
        "## Indexing a Pandas dataframe\n",
        "\n",
        "Sometimes we have the documents that we want to index in memory. Terrier makes it easy to index standard Python data structures, particularly [Pandas dataframes](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html).\n",
        "\n",
        "To do thise, we can use a `pt.DFIndexer()` object"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vw-4NIjlUY16",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ebc0b75e-3449-4e38-c6e2-01ddd77f5de9"
      },
      "source": [
        "import pandas as pd\n",
        "!rm -rf ./pd_index\n",
        "pd_indexer = pt.DFIndexer(\"./pd_index\")\n",
        "\n",
        "# optionally modify properties\n",
        "# index_properies = {\"block.indexing\":\"true\", \"invertedfile.lexiconscanner\":\"pointers\"}\n",
        "# indexer.setProperties(**index_properies)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "IndexingType.CLASSIC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NNkn2SdsPzBQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.DataFrame({ \n",
        "'docno':\n",
        "['1', '2', '3'],\n",
        "'url': \n",
        "['url1', 'url2', 'url3'],\n",
        "'text': \n",
        "['He ran out of money, so he had to stop playing',\n",
        "'The waves were crashing on the shore; it was a',\n",
        "'The body may perhaps compensates for the loss']\n",
        "})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vn-2M9iFyW9K",
        "colab_type": "code",
        "outputId": "7375299a-c13d-42fc-d2cd-34aecdbbee4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        }
      },
      "source": [
        "df"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>docno</th>\n",
              "      <th>url</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>url1</td>\n",
              "      <td>He ran out of money, so he had to stop playing</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>url2</td>\n",
              "      <td>The waves were crashing on the shore; it was a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>url3</td>\n",
              "      <td>The body may perhaps compensates for the loss</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  docno   url                                            text\n",
              "0     1  url1  He ran out of money, so he had to stop playing\n",
              "1     2  url2  The waves were crashing on the shore; it was a\n",
              "2     3  url3   The body may perhaps compensates for the loss"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K-idBQ5OUV9c",
        "colab_type": "text"
      },
      "source": [
        "Then there are a number of options to index the dataframe:    \n",
        "The first argument should always a pandas.Series object of Strings, which specifies the body of each document.    \n",
        "Any arguments after that are for specifying metadata.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rBXHWO4yUJT7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# no metadata\n",
        "# pd_indexer.index(df[\"text\"])\n",
        "\n",
        "# Add metadata fields as Pandas.Series objects, with the name of the Series object becoming the name of the meta field.\n",
        "indexref2 = pd_indexer.index(df[\"text\"], df[\"docno\"])\n",
        "# pd_indexer.index(df[\"text\"], df[\"docno\"], df[\"url\"])\n",
        "\n",
        "# Add metadata fields as lists to a keyword arguement\n",
        "# pd_indexer.index(df[\"text\"], docno=[\"1\",\"2\",\"3\"], url=[\"url1\", \"url2\", \"url3\"])\n",
        "\n",
        "# Add the metadata fields with a dictionary\n",
        "# meta_fields={\"docno\":[\"1\",\"2\",\"3\"],\"url\":[\"url1\", \"url2\", \"url3\"]}\n",
        "# pd_indexer.index(df[\"text\"], **meta_fields)\n",
        "\n",
        "# Add the entire dataframe as metadata\n",
        "# pd_indexer.index(df[\"text\"], df)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zjSUuBcou_L2",
        "colab_type": "text"
      },
      "source": [
        "## Indexing a iterable, generator, etc.\n",
        "\n",
        "You may not want to load all documents into memory, particularly for large collections. Terrier can index iterable objects (e.g., generators) that yield `dict` objects.\n",
        "\n",
        "To do thise, we can use a `pt.IterDictIndexer()` object. By default, `text` will be indexed and `docno` will be stored in the meta index. These can be configured with the `fields` and `meta` parameters, respectively."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CBYEKxK4u_uV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "f1880b9e-495e-44f2-a094-dbbf5ae7d1d9"
      },
      "source": [
        "# As an example, we will stream the ANTIQUE collection.\n",
        "# It is formatted as \"[docno] \\t [text] \\n\"\n",
        "import urllib\n",
        "import io\n",
        "def antique_doc_iter():\n",
        "    stream = urllib.request.urlopen('https://ciir.cs.umass.edu/downloads/Antique/antique-collection.txt')\n",
        "    stream = io.TextIOWrapper(stream)\n",
        "    for i, line in enumerate(stream):\n",
        "        if i % 100000 == 0:\n",
        "            print(f'processing document {i}')\n",
        "        docno, text = line.rstrip().split('\\t')\n",
        "        yield {'docno': docno, 'text': text}\n",
        "\n",
        "!rm -rf ./iter_index\n",
        "iter_indexer = pt.IterDictIndexer(\"./iter_index\")\n",
        "\n",
        "doc_iter = antique_doc_iter()\n",
        "indexref3 = iter_indexer.index(doc_iter)\n",
        "\n",
        "# Additional fields can be added in the dict. You can configure which fields are\n",
        "# indexed and which are used as metadata with the fields and meta parameters.\n",
        "# yield {'docno': docno, 'title': title, 'text': text, 'url': url}\n",
        "# iter_indexer.index(doc_iter, fields=['text', 'title'], meta=['docno', 'url'])"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "processing document 0\n",
            "processing document 100000\n",
            "processing document 200000\n",
            "processing document 300000\n",
            "processing document 400000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IKrZNyaUXRMg",
        "colab_type": "text"
      },
      "source": [
        "## Retrieval\n",
        "\n",
        "Lets see how we can use one of these for retrieval. Retrieval takes place using the `BatchRetrieve` object, by invoking `transform()` method for one or more queries. For a quick test, you can give just pass your query to `transform()`. \n",
        "\n",
        "BatchRetrieve will return the results as a Pandas dataframe.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zWj6YnN4Wnsz",
        "colab_type": "code",
        "outputId": "11b3b274-2921-4663-f91b-9ef711376582",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        }
      },
      "source": [
        "pt.BatchRetrieve(indexref).search(\"mathematical\")"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>docid</th>\n",
              "      <th>docno</th>\n",
              "      <th>rank</th>\n",
              "      <th>score</th>\n",
              "      <th>query</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>5040</td>\n",
              "      <td>5041</td>\n",
              "      <td>0</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>303</td>\n",
              "      <td>304</td>\n",
              "      <td>1</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>3534</td>\n",
              "      <td>3535</td>\n",
              "      <td>2</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>2444</td>\n",
              "      <td>2445</td>\n",
              "      <td>3</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>5011</td>\n",
              "      <td>5012</td>\n",
              "      <td>4</td>\n",
              "      <td>3.564534</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>147</th>\n",
              "      <td>1</td>\n",
              "      <td>7283</td>\n",
              "      <td>7284</td>\n",
              "      <td>147</td>\n",
              "      <td>2.834784</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148</th>\n",
              "      <td>1</td>\n",
              "      <td>6714</td>\n",
              "      <td>6715</td>\n",
              "      <td>148</td>\n",
              "      <td>2.811375</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>149</th>\n",
              "      <td>1</td>\n",
              "      <td>4746</td>\n",
              "      <td>4747</td>\n",
              "      <td>149</td>\n",
              "      <td>2.790373</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>150</th>\n",
              "      <td>1</td>\n",
              "      <td>8622</td>\n",
              "      <td>8623</td>\n",
              "      <td>150</td>\n",
              "      <td>2.759409</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>151</th>\n",
              "      <td>1</td>\n",
              "      <td>9800</td>\n",
              "      <td>9801</td>\n",
              "      <td>151</td>\n",
              "      <td>2.671493</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>152 rows × 6 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    qid  docid docno  rank     score         query\n",
              "0     1   5040  5041     0  3.566201  mathematical\n",
              "1     1    303   304     1  3.566201  mathematical\n",
              "2     1   3534  3535     2  3.566201  mathematical\n",
              "3     1   2444  2445     3  3.566201  mathematical\n",
              "4     1   5011  5012     4  3.564534  mathematical\n",
              "..   ..    ...   ...   ...       ...           ...\n",
              "147   1   7283  7284   147  2.834784  mathematical\n",
              "148   1   6714  6715   148  2.811375  mathematical\n",
              "149   1   4746  4747   149  2.790373  mathematical\n",
              "150   1   8622  8623   150  2.759409  mathematical\n",
              "151   1   9800  9801   151  2.671493  mathematical\n",
              "\n",
              "[152 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oK_wOzQrqi8t",
        "colab_type": "text"
      },
      "source": [
        "However, most IR experiments, will use a set of queries. You can pass such a set using a data frame for input."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dgtauIOAXGFY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "6a4e7e45-eaa4-48a0-d069-b271e3554826"
      },
      "source": [
        "import pandas as pd\n",
        "topics = pd.DataFrame([[\"2\", \"mathematical\"]],columns=['qid','query'])\n",
        "pt.BatchRetrieve(indexref).transform(topics)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>docid</th>\n",
              "      <th>docno</th>\n",
              "      <th>rank</th>\n",
              "      <th>score</th>\n",
              "      <th>query</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>5040</td>\n",
              "      <td>5041</td>\n",
              "      <td>0</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>303</td>\n",
              "      <td>304</td>\n",
              "      <td>1</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>3534</td>\n",
              "      <td>3535</td>\n",
              "      <td>2</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>2444</td>\n",
              "      <td>2445</td>\n",
              "      <td>3</td>\n",
              "      <td>3.566201</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>5011</td>\n",
              "      <td>5012</td>\n",
              "      <td>4</td>\n",
              "      <td>3.564534</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>147</th>\n",
              "      <td>2</td>\n",
              "      <td>7283</td>\n",
              "      <td>7284</td>\n",
              "      <td>147</td>\n",
              "      <td>2.834784</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148</th>\n",
              "      <td>2</td>\n",
              "      <td>6714</td>\n",
              "      <td>6715</td>\n",
              "      <td>148</td>\n",
              "      <td>2.811375</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>149</th>\n",
              "      <td>2</td>\n",
              "      <td>4746</td>\n",
              "      <td>4747</td>\n",
              "      <td>149</td>\n",
              "      <td>2.790373</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>150</th>\n",
              "      <td>2</td>\n",
              "      <td>8622</td>\n",
              "      <td>8623</td>\n",
              "      <td>150</td>\n",
              "      <td>2.759409</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>151</th>\n",
              "      <td>2</td>\n",
              "      <td>9800</td>\n",
              "      <td>9801</td>\n",
              "      <td>151</td>\n",
              "      <td>2.671493</td>\n",
              "      <td>mathematical</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>152 rows × 6 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    qid  docid docno  rank     score         query\n",
              "0     2   5040  5041     0  3.566201  mathematical\n",
              "1     2    303   304     1  3.566201  mathematical\n",
              "2     2   3534  3535     2  3.566201  mathematical\n",
              "3     2   2444  2445     3  3.566201  mathematical\n",
              "4     2   5011  5012     4  3.564534  mathematical\n",
              "..   ..    ...   ...   ...       ...           ...\n",
              "147   2   7283  7284   147  2.834784  mathematical\n",
              "148   2   6714  6715   148  2.811375  mathematical\n",
              "149   2   4746  4747   149  2.790373  mathematical\n",
              "150   2   8622  8623   150  2.759409  mathematical\n",
              "151   2   9800  9801   151  2.671493  mathematical\n",
              "\n",
              "[152 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XLcNP0XHqwQs",
        "colab_type": "text"
      },
      "source": [
        "Thats the end of the indexing tutorial - you can continue with other example tutorials."
      ]
    }
  ]
}