iirc_create_indices.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating indices for IIRC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "test_set = json.load(open('data/iirc_test.json','r'))\n",
    "context_articles = json.load(open(\"data/context_articles.json\",'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9th paratroopers assault regiment \"col moschin\"\n",
      "goldfinger (film)\n",
      "list of international cricket council members\n",
      "icc americas championship\n",
      "the rev\n",
      "avenged sevenfold\n",
      "fox footy\n",
      "herald sun\n",
      "fox footy\n",
      "herald sun\n",
      "united states\n",
      "judeo-iraqi arabic\n",
      "maya civilization\n",
      "black watch\n",
      "suicidal tendencies\n",
      "western hockey league\n",
      "national hockey league\n",
      "home run\n",
      "minor league baseball\n",
      "colonel\n",
      "colonel\n",
      "massachusetts institute of technology\n",
      "israel\n",
      "harvard business review\n",
      "american football\n",
      "college football\n",
      "united states\n",
      "billboard 200\n",
      "romeo discography\n",
      "billboard 200\n",
      "master p\n",
      "hip hop history\n",
      "billboard 200\n",
      "louisiana\n",
      "arizona\n",
      "state farm stadium\n",
      "louisiana\n",
      "united states\n",
      "gulf of mexico\n",
      "saffir–simpson scale\n",
      "forgotten realms\n",
      "list of dungeons & dragons rulebooks\n",
      "mexico\n",
      "napoleon iii\n",
      "american football\n",
      "national football league\n",
      "mexico\n",
      "lucha libre\n",
      "protagonist\n",
      "double dragon\n",
      "world war ii\n",
      "banff, alberta\n",
      "american football\n",
      "quarterback\n",
      "college football\n",
      "2009 nfl draft\n",
      "new york city\n",
      "los angeles\n",
      "metal massacre\n",
      "metal massacre\n",
      "hull city a.f.c.\n",
      "port phillip district\n",
      "slant magazine\n",
      "everton f.c.\n",
      "everton f.c.\n",
      "florence\n",
      "soviet union\n",
      "world hockey association\n",
      "fa cup\n",
      "mets de guaynabo (basketball)\n",
      "mets de guaynabo (basketball)\n",
      "mario morales coliseum\n",
      "mario morales coliseum\n",
      "guaynabo fluminense fc\n",
      "puerto rico soccer league\n",
      "guaynabo fluminense fc\n",
      "recording industry association of america\n",
      "united kingdom\n",
      "fc kariya\n",
      "south australia\n",
      "adelaide\n",
      "ice hockey\n",
      "canada\n",
      "university of toronto\n",
      "the pittsburgh press\n",
      "royal navy\n",
      "chicago\n",
      "rock and roll\n",
      "spanish civil war\n",
      "boston brawlers\n",
      "luanda\n",
      "sony\n",
      "florida\n",
      "florida public service commission\n",
      "world series\n",
      "national league\n",
      "mediterranean fleet\n",
      "member of parliament\n",
      "bruce lee\n",
      "shandong\n",
      "continental divide of the americas\n",
      "shankh monastery\n",
      "the emperor jones\n",
      "france\n",
      "napoleon\n",
      "belgium\n",
      "detroit pistons\n",
      "english channel\n",
      "royal australian navy\n",
      "republic of china navy\n",
      "order of the red banner\n",
      "harper's magazine\n",
      "morris museum of art\n",
      "midwestern united states\n",
      "french union\n",
      "sex organ\n",
      "dismissal of james comey\n",
      "national basketball association\n",
      "omaha, nebraska\n",
      "birmingham city f.c.\n",
      "australia\n",
      "cape town\n",
      "jerusalem\n",
      "football league second division\n",
      "q (magazine)\n",
      "r.e.m.\n",
      "new york city\n",
      "canada\n",
      "southampton\n",
      "the bahamas\n",
      "quebec\n",
      "ministry of commerce and industry (india)\n",
      "licence raj\n",
      "river thames\n",
      "fishman (wrestler)\n",
      "athens\n",
      "professional wrestling\n",
      "professional wrestling match types\n",
      "rob van dam\n",
      "professional wrestling authority figures\n",
      "bury f.c.\n",
      "liverpool f.c.\n",
      "minor counties of english and welsh cricket\n",
      "first-class cricket\n",
      "world war ii\n",
      "democratic party (united states)\n",
      "pop music\n",
      "slant magazine\n",
      "g (musical note)\n",
      "europe\n",
      "world war i\n",
      "detroit\n",
      "world war ii\n",
      "english channel\n",
      "river thames\n",
      "canada\n",
      "provinces and territories of canada\n",
      "germany\n",
      "bbc\n",
      "member of parliament\n",
      "karl marx\n",
      "fascism\n",
      "bbc\n",
      "dallas cowboys\n",
      "national football league\n",
      "jason witten\n",
      "italy national football team\n",
      "usl championship\n",
      "a-league (1995–2004)\n",
      "national professional soccer league (1984–2001)\n",
      "harvard university\n",
      "professor\n",
      "michigan state university\n",
      "algeria\n",
      "world war i\n",
      "television pilot\n",
      "national league\n",
      "2004 national league division series\n",
      "2004 national league championship series\n",
      "world series\n",
      "world war ii\n",
      "united states air force\n",
      "cold war\n",
      "gulf war\n",
      "soviet union\n",
      "lockheed c-141 starlifter\n",
      "lockheed c-130 hercules\n",
      "dover air force base\n",
      "gymnasium (school)\n",
      "dubrovnik\n",
      "harvard university\n",
      "bonwit teller\n",
      "europe\n",
      "siberia\n",
      "canada\n",
      "people of the wolf\n",
      "american broadcasting company\n",
      "lieutenant\n",
      "communist party of china\n",
      "soviet union\n",
      "moscow\n",
      "type ix submarine\n",
      "fifa world cup\n",
      "1994 fifa world cup\n",
      "cleveland international records\n",
      "grammy award\n",
      "grammy award\n",
      "cleveland international records\n",
      "joey miskulin\n",
      "grammy award\n",
      "paul oakenfold\n",
      "r.e.m.\n",
      "sputnikmusic\n",
      "compilation album\n",
      "extended play\n",
      "single (music)\n",
      "british phonographic industry\n",
      "billboard 200\n",
      "united states\n",
      "variety (magazine)\n",
      "canada\n",
      "pulp fiction\n",
      "american hockey league\n",
      "goals against average\n",
      "lawyer\n",
      "english language\n",
      "napoleon\n",
      "battle of kulm\n",
      "extended play\n",
      "emi\n",
      "billboard hot 100\n",
      "overhead camshaft\n",
      "eugene o'neill\n",
      "philadelphia\n",
      "new york city\n",
      "dean (education)\n",
      "chicago\n",
      "egypt\n",
      "atlantic ocean\n",
      "indian ocean\n",
      "american football\n",
      "balkans\n",
      "jews\n",
      "world war ii\n",
      "first-class cricket\n",
      "batting average (cricket)\n",
      "bowling average\n",
      "west germany\n",
      "catholic church\n",
      "kolkata\n",
      "soap opera\n",
      "united states\n",
      "the a-team\n",
      "brush with danger\n",
      "howard university\n",
      "japan\n",
      "the new york times\n",
      "obie award\n",
      "liverpool\n",
      "philadelphia\n",
      "english language\n",
      "democratic party (united states)\n",
      "republican party (united states)\n",
      "methodism\n",
      "glossary of professional wrestling terms\n",
      "sacramento, california\n",
      "wwe\n",
      "wwe tough enough\n",
      "professional wrestling match types\n",
      "scotland\n",
      "jerusalem\n",
      "sea of galilee\n",
      "stanley cup\n",
      "boston bruins\n",
      "boston bruins\n",
      "buffalo sabres\n",
      "national hockey league\n",
      "american hockey league\n",
      "international hockey league (1945–2001)\n",
      "world hockey association\n",
      "wwe nxt\n",
      "wwe raw\n",
      "wwe intercontinental championship\n",
      "2014 fig rhythmic gymnastics world cup series\n",
      "2014 fig rhythmic gymnastics world cup series\n",
      "rhythmic gymnastics grand prix\n",
      "melbourne\n",
      "lyon\n",
      "venice\n",
      "rome\n",
      "naples\n",
      "florence\n",
      "tampa bay rowdies (1975–93)\n",
      "united states\n",
      "united states army\n",
      "united states navy\n",
      "spanish navy\n",
      "soviet union\n",
      "egypt\n",
      "master of arts\n",
      "queen's university belfast\n",
      "world war ii\n",
      "iași\n",
      "alexandru ioan cuza university\n",
      "university of peshawar\n",
      "bachelor of laws\n",
      "bbc\n",
      "wii\n",
      "los angeles dodgers\n",
      "pac-12 conference\n",
      "billboard 200\n",
      "billboard (magazine)\n",
      "billboard hot 100\n",
      "australia\n",
      "national basketball association\n",
      "dallas mavericks\n",
      "indiana pacers\n",
      "cleveland cavaliers\n",
      "world war ii\n",
      "malta\n",
      "port said\n",
      "alexandria\n",
      "tobruk\n",
      "washington, d.c.\n",
      "national league (division)\n",
      "efl league two\n",
      "northampton town f.c.\n",
      "premier league\n",
      "birmingham city f.c.\n",
      "west bromwich albion f.c.\n",
      "fa cup\n",
      "sheffield wednesday f.c.\n",
      "wembley stadium\n",
      "everton f.c.\n",
      "disc jockey\n",
      "michael jackson\n",
      "fjord horse\n",
      "sunniva\n",
      "germany\n",
      "mississippi river\n",
      "st. louis\n",
      "earned run average\n",
      "innings pitched\n",
      "batting average (baseball)\n",
      "perth\n",
      "saint patrick's day\n",
      "university of michigan\n",
      "associated press\n",
      "nebraska cornhuskers football\n",
      "liverpool f.c.\n",
      "thessaloniki\n",
      "tbilisi\n",
      "iraq\n",
      "itv granada\n",
      "united states\n",
      "sydney\n",
      "first-class cricket\n",
      "world war i\n",
      "efl league two\n",
      "scunthorpe united f.c.\n",
      "national cartoonists society\n",
      "peabody award\n",
      "emmy award\n",
      "time (magazine)\n",
      "belgian first division a\n",
      "list of legends of tomorrow characters\n",
      "list of legends of tomorrow characters\n",
      "list of legends of tomorrow characters\n",
      "list of legends of tomorrow characters\n",
      "list of legends of tomorrow characters\n",
      "list of legends of tomorrow characters\n",
      "list of legends of tomorrow characters\n",
      "list of legends of tomorrow characters\n",
      "menhir\n",
      "malta\n",
      "nascar cup series\n",
      "nascar gander rv & outdoors truck series\n",
      "nascar\n",
      "seating capacity\n",
      "world war ii\n",
      "united states\n",
      "showtime (tv network)\n",
      "paris\n",
      "rome\n",
      "montreal wanderers\n",
      "world war i\n",
      "great depression\n",
      "world war ii\n",
      "maryland\n",
      "midwestern united states\n",
      "arkansas\n",
      "music video\n",
      "st. louis\n",
      "frigate\n",
      "philippine–american war\n",
      "north america\n",
      "united states\n",
      "canada\n",
      "canada\n",
      "united states\n",
      "quebec\n",
      "northwest territory\n",
      "everton f.c.\n",
      "loan (sports)\n",
      "national hockey league\n",
      "new york rangers\n",
      "new york rangers\n",
      "national hockey league\n",
      "american hockey league\n",
      "gulf of mexico\n",
      "united states\n",
      "washington, d.c.\n",
      "british columbia conservative party\n",
      "majority government\n",
      "duke university\n",
      "time (magazine)\n",
      "sweden\n",
      "bbc\n",
      "united states house of representatives\n",
      "broadway theatre\n",
      "pop music\n",
      "florida\n",
      "tropical disease\n",
      "washington, d.c.\n",
      "thessaloniki\n",
      "byzantine empire\n",
      "exarchate of ravenna\n",
      "bollywood\n",
      "lahore\n",
      "columbia river\n",
      "washington (state)\n",
      "cold war\n",
      "st. louis\n",
      "catholic church\n",
      "university of notre dame\n",
      "united states navy\n",
      "new york city\n",
      "japan\n",
      "rudolph schindler (architect)\n",
      "oman\n",
      "first-class cricket\n",
      "stanford 20/20\n",
      "2008 stanford super series\n",
      "government of canada\n",
      "united nations\n",
      "stanford university\n",
      "university of california, berkeley\n",
      "harvard university\n",
      "north america\n",
      "united nations\n",
      "new york city\n",
      "wework\n",
      "wework\n",
      "wework\n",
      "rhône group\n",
      "wework\n",
      "wework\n",
      "bachelor of arts\n",
      "master of arts\n",
      "sheffield united f.c.\n",
      "crust (geology)\n",
      "united states house of representatives\n",
      "national science foundation\n",
      "2014 scottish league cup final\n",
      "world war ii\n",
      "middle east\n",
      "ukraine\n",
      "saint petersburg\n",
      "south china sea\n",
      "vietnam war\n",
      "deacon\n",
      "second french intervention in mexico\n",
      "mexican–american war\n",
      "catholic church\n",
      "benito juárez\n",
      "united kingdom\n",
      "france\n",
      "spain\n",
      "napoleon iii\n",
      "austria-hungary\n",
      "ukraine\n",
      "world war i\n",
      "germany\n",
      "university of vienna\n",
      "the oprah winfrey show\n",
      "wkbw-tv\n",
      "trieste\n",
      "transylvania\n",
      "austria-hungary\n",
      "university of bucharest\n",
      "international criminal court\n",
      "scunthorpe united f.c.\n",
      "napoleon iii\n",
      "wnew-fm\n",
      "harvard university\n",
      "republican party (united states)\n",
      "greenhouse gas\n",
      "american broadcasting company\n",
      "pharaoh\n",
      "egyptology\n",
      "leipzig\n",
      "egypt\n",
      "constantinople\n",
      "black sea\n",
      "caribbean\n",
      "new york city\n",
      "san diego\n",
      "pearl harbor\n",
      "mtv\n",
      "music video\n",
      "western hockey league\n",
      "omaha, nebraska\n",
      "los angeles\n",
      "soap opera\n",
      "regis philbin\n",
      "talk show\n",
      "edge (wrestler)\n",
      "the undertaker\n",
      "battle royal (professional wrestling)\n",
      "professional wrestling match types\n",
      "gymnasium (school)\n",
      "kraków\n",
      "united states dollar\n",
      "pay-per-view\n",
      "circuito de jerez\n",
      "mika häkkinen\n",
      "mclaren\n",
      "david coulthard\n",
      "williams grand prix engineering\n",
      "jacques villeneuve\n",
      "venice\n",
      "malta\n",
      "palm springs, california\n",
      "united states\n",
      "oregon\n",
      "vietnam war\n",
      "major\n",
      "mediterranean sea\n",
      "neolithic\n",
      "aristotle\n",
      "theophrastus\n",
      "serie a\n",
      "uefa champions league\n",
      "uefa super cup\n",
      "coppa italia\n",
      "lord voldemort\n",
      "lord voldemort\n",
      "prophecy\n",
      "harry potter\n",
      "list of supporting harry potter characters\n",
      "university of michigan\n",
      "columbia university\n",
      "university of cambridge\n",
      "university of michigan\n",
      "marylebone\n",
      "christ church, oxford\n",
      "first-class cricket\n",
      "oxford university cricket club\n",
      "marylebone cricket club\n",
      "magdalen ground\n",
      "batting average (cricket)\n",
      "history of english amateur cricket\n",
      "sydney\n",
      "justice of the peace\n",
      "newington college\n",
      "saint petersburg\n",
      "pompeii\n",
      "ottawa rebel\n",
      "buffalo bandits\n",
      "billboard hot 100\n",
      "music video\n",
      "billboard 200\n",
      "saturday night live\n",
      "chuck e.'s in love\n",
      "time (magazine)\n",
      "manchester united f.c.\n",
      "metacritic\n",
      "stephen thomas erlewine\n",
      "rolling stone\n",
      "slant magazine\n",
      "columbus blue jackets\n",
      "tampa, florida\n",
      "fredrik modin\n",
      "ottoman empire\n",
      "europe\n",
      "indian ocean\n",
      "somalia\n",
      "south africa\n",
      "first-class cricket\n",
      "batting average (baseball)\n",
      "baltimore orioles\n",
      "batting average (baseball)\n",
      "home run\n",
      "run batted in\n",
      "american football\n",
      "national football league\n",
      "college football\n",
      "philadelphia eagles\n",
      "world war ii\n",
      "catholic church\n",
      "palatinate (region)\n",
      "france\n",
      "galicia (eastern europe)\n",
      "netherlands\n",
      "poland\n",
      "switzerland\n",
      "the new york times\n",
      "the pittsburgh press\n",
      "kanye west\n",
      "gibraltar\n",
      "east germany\n",
      "socialist unity party of germany\n",
      "volkskammer\n",
      "norwich city f.c.\n",
      "mansfield town f.c.\n",
      "the tonight show\n",
      "bronze age\n",
      "iron age\n",
      "san diego sockers (1978–96)\n",
      "national professional soccer league (1984–2001)\n",
      "anterior cruciate ligament\n",
      "north carolina fusion u23\n",
      "usl championship\n",
      "milwaukee wave\n",
      "a-league (1995–2004)\n",
      "minnesota thunder\n",
      "san diego flash\n",
      "third base\n",
      "third base\n",
      "pitcher\n",
      "run batted in\n",
      "batting average (baseball)\n",
      "palm springs, california\n",
      "nme\n",
      "extended play\n",
      "virginia\n",
      "ancient rome\n",
      "montreal canadiens\n",
      "nbc\n",
      "fulbright program\n",
      "university of chicago\n",
      "vietnam war\n",
      "native americans in the united states\n",
      "native americans in the united states\n",
      "native americans in the united states\n",
      "california\n",
      "washington, d.c.\n",
      "united states\n",
      "washington, d.c.\n",
      "native americans in the united states\n",
      "wwe\n",
      "ohio valley wrestling\n",
      "wwe divas championship\n",
      "mahir çayan\n",
      "nato\n",
      "martial law\n",
      "karl marx\n",
      "british columbia\n",
      "canada\n",
      "united states\n",
      "washington, d.c.\n",
      "new york city\n",
      "united states army\n",
      "second lieutenant\n",
      "philippines\n",
      "spanish–american war\n",
      "new york city\n",
      "brown university\n",
      "wesleyan university\n",
      "washington, d.c.\n",
      "washington, d.c.\n",
      "toronto\n",
      "ontario\n",
      "the new york times\n",
      "jazz\n",
      "catholic church\n",
      "feast of the cross\n",
      "constantinople\n",
      "constitution of the united states\n",
      "vietnam\n",
      "south china sea\n",
      "saqqara\n",
      "thebes, egypt\n",
      "north sea\n",
      "british museum\n",
      "florence\n",
      "paris\n",
      "london\n",
      "broadway theatre\n",
      "world war ii\n",
      "saint petersburg\n",
      "stockholm\n",
      "hollywood\n",
      "efl cup\n",
      "premier league\n",
      "west ham united f.c.\n",
      "plymouth argyle f.c.\n",
      "bury f.c.\n",
      "plymouth argyle f.c.\n",
      "efl trophy\n",
      "portsmouth f.c.\n",
      "torquay united f.c.\n",
      "dartmouth college\n",
      "los angeles\n",
      "list of discontinued volkswagen group petrol engines\n",
      "twin-turbo\n",
      "roy (film)\n",
      "derby county f.c.\n",
      "football league second division\n",
      "northampton town f.c.\n",
      "newcastle united f.c.\n",
      "sunderland a.f.c.\n",
      "russian empire\n",
      "jews\n",
      "bachelor of arts\n",
      "moscow state university\n",
      "baku state university\n",
      "fulbright program\n",
      "stanford university\n",
      "university of california, berkeley\n",
      "university of california, los angeles\n",
      "harvard university\n",
      "columbia university\n",
      "united states\n",
      "composer\n",
      "vienna\n",
      "fulbright program\n",
      "rome prize\n",
      "indian council of medical research\n",
      "department of biotechnology\n",
      "council of scientific and industrial research\n",
      "special olympics world games\n",
      "u2\n",
      "raidió teilifís éireann\n",
      "bbc\n",
      "cbc television\n",
      "europe\n",
      "kent\n",
      "battle of austerlitz\n",
      "palermo\n",
      "world war i\n",
      "alaska\n",
      "bachelor of arts\n",
      "political science\n",
      "kenya\n",
      "yale law school\n",
      "university of oxford\n",
      "national institute for health and care excellence\n",
      "s.l. benfica\n",
      "farm team\n",
      "spring training\n",
      "milwaukee brewers\n",
      "world war ii\n",
      "united states army\n",
      "japan\n",
      "communist party of china\n",
      "chicago\n",
      "wisconsin\n",
      "los angeles\n",
      "australia\n",
      "united kingdom\n",
      "united kingdom\n",
      "sex pistols\n",
      "fourth crusade\n",
      "protestantism\n",
      "second vatican council\n",
      "li mi (republic of china general)\n",
      "chiang kai-shek\n",
      "nanjing\n",
      "adolf hitler\n",
      "hat-trick\n",
      "brown university\n",
      "harvard university\n",
      "bachelor of laws\n",
      "london\n",
      "bbc\n",
      "japan\n",
      "incest\n",
      "david o. russell\n",
      "royal air force\n",
      "kayfabe\n",
      "italy national football team\n",
      "republic of ireland national football team\n",
      "atletico roma f.c.\n",
      "list of the adventures of tintin characters\n",
      "list of the adventures of tintin characters\n",
      "north sea\n",
      "list of the adventures of tintin characters\n",
      "connecticut\n",
      "united states\n",
      "germany\n",
      "moscow\n",
      "saint petersburg\n",
      "montreal canadiens\n",
      "vancouver canucks\n",
      "2008–09 nhl season\n",
      "montreal canadiens\n",
      "new york americans\n",
      "ottawa senators (original)\n",
      "sydney\n",
      "uk singles chart\n",
      "uk albums chart\n",
      "uk albums chart\n",
      "saint petersburg\n",
      "world war i\n",
      "constantinople\n",
      "english civil war\n",
      "church of england\n",
      "serie b\n",
      "dublin\n",
      "british army\n",
      "1948 arab–israeli war\n",
      "parish church\n",
      "anglo-catholicism\n",
      "san francisco\n",
      "japan\n",
      "battleship\n",
      "aircraft carrier\n",
      "san francisco\n",
      "fiji\n",
      "peruvian nationalist party\n",
      "anabaptism\n",
      "mennonites\n",
      "west prussia\n",
      "vistula\n",
      "snow patrol\n",
      "huddersfield town a.f.c.\n",
      "premier league\n",
      "fulham f.c.\n",
      "milwaukee brewers\n",
      "free agent\n",
      "minor league baseball\n",
      "minor league baseball\n",
      "university of cambridge\n",
      "world war ii\n",
      "dna\n",
      "hydrogen bond\n",
      "dna\n",
      "university of cambridge\n",
      "athens\n",
      "ottoman empire\n",
      "united states\n",
      "french indochina\n",
      "french protectorate of laos\n",
      "st aldate's church\n",
      "christ church, oxford\n",
      "bachelor of arts\n",
      "master of arts\n",
      "bachelor of divinity\n",
      "doctor of divinity\n",
      "university of oxford\n",
      "war of the first coalition\n",
      "french first republic\n",
      "war of the first coalition\n",
      "belgium\n",
      "france\n",
      "pink floyd\n",
      "heavy metal music\n",
      "rush (band)\n",
      "punk rock\n",
      "bad brains\n",
      "biology\n",
      "chemistry\n",
      "ukraine\n",
      "tirana\n",
      "fc metalurh donetsk\n",
      "lazar popović\n",
      "independent politician\n",
      "poland\n",
      "poland\n",
      "france\n",
      "world war i\n",
      "paris\n",
      "association football\n",
      "birmingham city f.c.\n",
      "cardiff city f.c.\n",
      "huddersfield town a.f.c.\n",
      "second vatican council\n",
      "latin\n",
      "jesus\n",
      "major (united kingdom)\n",
      "south africa\n",
      "world war i\n",
      "order of the british empire\n",
      "marylebone cricket club\n",
      "aide-de-camp\n",
      "napoleon\n",
      "french invasion of russia\n",
      "vistula\n",
      "battle of leipzig\n",
      "hyderabad state\n",
      "jawaharlal nehru\n",
      "red army\n",
      "soviet union\n",
      "treaty of versailles\n",
      "french revolutionary wars\n",
      "mediterranean sea\n",
      "royal shakespeare company\n",
      "miley stewart\n",
      "hannah montana (soundtrack)\n",
      "pop rock\n",
      "england national football team\n",
      "efl cup\n",
      "leeds united f.c.\n",
      "caribbean\n",
      "lisbon\n",
      "spain\n",
      "france\n",
      "stockholm\n",
      "romanian bridgehead\n",
      "hotchkiss h35\n",
      "nbc\n",
      "history of the united states\n",
      "university of oxford\n",
      "emeritus\n",
      "university of california, los angeles\n",
      "fc barcelona\n",
      "the daily telegraph\n",
      "the beatles\n",
      "elton john\n",
      "nme\n",
      "dj shadow\n",
      "radiohead\n",
      "r.e.m.\n",
      "allmusic\n",
      "hi-fi news & record review\n",
      "stylus magazine\n",
      "world war ii\n",
      "india\n",
      "tibet\n",
      "hanoi\n",
      "hanoi\n",
      "shandong\n",
      "henan\n",
      "oxford\n",
      "uk singles chart\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "documents = []\n",
    "all_titles = []\n",
    "\n",
    "for item in test_set:\n",
    "    if item['title'].lower() not in all_titles:\n",
    "        documents.append({\n",
    "                \"title\": item['title'],\n",
    "                \"content\": item[\"text\"]\n",
    "            }\n",
    "        )\n",
    "        all_titles.append(item['title'].lower())\n",
    "    for link in item[\"links\"]:\n",
    "        if link['target'].lower() in context_articles and link['target'].lower() not in all_titles:\n",
    "            documents.append({\n",
    "                \"title\": link['target'],\n",
    "                \"content\": context_articles[link['target'].lower()]\n",
    "            })\n",
    "            all_titles.append(link['target'].lower())\n",
    "        else:\n",
    "            print(link['target'].lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7028/7028 [01:53<00:00, 61.70it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import spacy\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "nlp.add_pipe(nlp.create_pipe(\"sentencizer\"))\n",
    "\n",
    "stride = 2\n",
    "max_length = 3\n",
    "\n",
    "def window(documents, stride=2, max_length=3):\n",
    "    treated_documents = []\n",
    "\n",
    "    for j,document in enumerate(tqdm(documents)):\n",
    "        doc_text = document['content']\n",
    "        doc = nlp(doc_text[:10000])\n",
    "        sentences = [sent.string.strip() for sent in doc.sents]\n",
    "        for i in range(0, len(sentences), stride):\n",
    "            segment = ' '.join(sentences[i:i + max_length])\n",
    "            treated_documents.append({\n",
    "                \"title\": document['title'],\n",
    "                \"contents\": document['title']+\". \"+segment,\n",
    "                \"segment\": segment\n",
    "            })\n",
    "            if i + max_length >= len(sentences):\n",
    "                break\n",
    "    return treated_documents\n",
    "\n",
    "treated_documents = window(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir data/iirc_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"data/iirc_indices/contents.jsonl\",'w')\n",
    "\n",
    "for i, doc in enumerate(treated_documents):\n",
    "    doc['id'] = i\n",
    "    if doc['segment'] != \"\":\n",
    "        f.write(json.dumps(doc)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance.\n",
      "2022-10-19 19:49:52,586 INFO  [main] index.IndexCollection (IndexCollection.java:650) - Setting log level to INFO\n",
      "2022-10-19 19:49:52,589 INFO  [main] index.IndexCollection (IndexCollection.java:653) - Starting indexer...\n",
      "2022-10-19 19:49:52,589 INFO  [main] index.IndexCollection (IndexCollection.java:654) - ============ Loading Parameters ============\n",
      "2022-10-19 19:49:52,589 INFO  [main] index.IndexCollection (IndexCollection.java:655) - DocumentCollection path: data/iirc_indices\n",
      "2022-10-19 19:49:52,590 INFO  [main] index.IndexCollection (IndexCollection.java:656) - CollectionClass: JsonCollection\n",
      "2022-10-19 19:49:52,590 INFO  [main] index.IndexCollection (IndexCollection.java:657) - Generator: DefaultLuceneDocumentGenerator\n",
      "2022-10-19 19:49:52,590 INFO  [main] index.IndexCollection (IndexCollection.java:658) - Threads: 1\n",
      "2022-10-19 19:49:52,591 INFO  [main] index.IndexCollection (IndexCollection.java:659) - Stemmer: porter\n",
      "2022-10-19 19:49:52,591 INFO  [main] index.IndexCollection (IndexCollection.java:660) - Keep stopwords? false\n",
      "2022-10-19 19:49:52,591 INFO  [main] index.IndexCollection (IndexCollection.java:661) - Stopwords:  null\n",
      "2022-10-19 19:49:52,592 INFO  [main] index.IndexCollection (IndexCollection.java:662) - Store positions? false\n",
      "2022-10-19 19:49:52,592 INFO  [main] index.IndexCollection (IndexCollection.java:663) - Store docvectors? false\n",
      "2022-10-19 19:49:52,592 INFO  [main] index.IndexCollection (IndexCollection.java:664) - Store document \"contents\" field? false\n",
      "2022-10-19 19:49:52,593 INFO  [main] index.IndexCollection (IndexCollection.java:665) - Store document \"raw\" field? true\n",
      "2022-10-19 19:49:52,593 INFO  [main] index.IndexCollection (IndexCollection.java:666) - Optimize (merge segments)? false\n",
      "2022-10-19 19:49:52,593 INFO  [main] index.IndexCollection (IndexCollection.java:667) - Whitelist: null\n",
      "2022-10-19 19:49:52,594 INFO  [main] index.IndexCollection (IndexCollection.java:687) - Directly building Lucene indexes...\n",
      "2022-10-19 19:49:52,594 INFO  [main] index.IndexCollection (IndexCollection.java:688) - Index path: data/iirc_index\n",
      "2022-10-19 19:49:52,596 INFO  [main] index.IndexCollection (IndexCollection.java:737) - ============ Indexing Collection ============\n",
      "2022-10-19 19:49:52,709 INFO  [main] index.IndexCollection (IndexCollection.java:798) - Thread pool with 1 threads initialized.\n",
      "2022-10-19 19:49:52,710 INFO  [main] index.IndexCollection (IndexCollection.java:800) - Initializing collection in data/iirc_indices\n",
      "2022-10-19 19:49:52,714 INFO  [main] index.IndexCollection (IndexCollection.java:803) - 1 file found\n",
      "2022-10-19 19:49:52,714 INFO  [main] index.IndexCollection (IndexCollection.java:804) - Starting to index...\n",
      "2022-10-19 19:50:08,500 DEBUG [pool-2-thread-1] index.IndexCollection$LocalIndexerThread (IndexCollection.java:239) - iirc_indices/contents.jsonl: 146126 docs added.\n",
      "2022-10-19 19:50:13,637 INFO  [main] index.IndexCollection (IndexCollection.java:888) - Indexing Complete! 146,126 documents indexed\n",
      "2022-10-19 19:50:13,638 INFO  [main] index.IndexCollection (IndexCollection.java:889) - ============ Final Counter Values ============\n",
      "2022-10-19 19:50:13,638 INFO  [main] index.IndexCollection (IndexCollection.java:890) - indexed:          146,126\n",
      "2022-10-19 19:50:13,638 INFO  [main] index.IndexCollection (IndexCollection.java:891) - unindexable:            0\n",
      "2022-10-19 19:50:13,638 INFO  [main] index.IndexCollection (IndexCollection.java:892) - empty:                  0\n",
      "2022-10-19 19:50:13,639 INFO  [main] index.IndexCollection (IndexCollection.java:893) - skipped:                0\n",
      "2022-10-19 19:50:13,639 INFO  [main] index.IndexCollection (IndexCollection.java:894) - errors:                 0\n",
      "2022-10-19 19:50:13,645 INFO  [main] index.IndexCollection (IndexCollection.java:897) - Total 146,126 documents indexed in 00:00:21\n"
     ]
    }
   ],
   "source": [
    "!python3 -m pyserini.index -collection JsonCollection -generator DefaultLuceneDocumentGenerator -threads 1 -input data/iirc_indices -index data/iirc_index -storeRaw"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}