data_stats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This notebook provides some exploration of the dataset itself.\n",
    "We will use these files in the dataset directory (you need to first download and unzip it):\n",
    "\n",
    "data_set_citations.json : each entry corresponds to a mention from a paper to a dataset, mention phrases are also given\n",
    "\n",
    "file/text/*.txt: all the text files of the papers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json, codecs, re\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## dataset directory\n",
    "DIR = \"../train_test/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "## function to extract all papers \n",
    "def _extract(dir_name='files/text/', extension='.txt'):\n",
    "    dir_name = DIR + dir_name\n",
    "    full_text = {}\n",
    "    for item in os.listdir(dir_name):\n",
    "        if item.endswith(extension):\n",
    "            file_name = os.path.abspath(dir_name + '/' + item)\n",
    "            with codecs.open(file_name, 'r') as f:\n",
    "                try:\n",
    "                    lines = f.readlines()\n",
    "                    #TODO document structure\n",
    "                    #text = ' '.join([s.strip() for s in lines])\n",
    "                    text = ' '.join([s.strip() for s in lines])\n",
    "                    text = re.sub('\\d', '0', text)\n",
    "                    text = re.sub('[^ ]- ', '', text)\n",
    "                    full_text[item] = text\n",
    "                except:\n",
    "                    pass\n",
    "    return full_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of publications given: 5000\n"
     ]
    }
   ],
   "source": [
    "data_set_citations = pd.read_json(DIR+'data_set_citations.json', encoding='utf-8')\n",
    "full_text = _extract()\n",
    "print ('total number of publications given:', len(full_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Although there are 5K papers given, not all of them are annotated. We will only use annotated data for training and evaluation. The annotations can be retrieved from data_set_citations. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_set_ids = data_set_citations['data_set_id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total entries:  5499\n"
     ]
    }
   ],
   "source": [
    "print ('Total entries: ', len(data_set_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of datasets:  1028\n"
     ]
    }
   ],
   "source": [
    "print ('Number of datasets: ', len(set(data_set_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of annotated papers:  2500\n"
     ]
    }
   ],
   "source": [
    "print ('Number of annotated papers: ', len(set(data_set_citations['publication_id'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## retreve full text of annotated papers\n",
    "publication_with_mentions = {}\n",
    "publication_with_mentions_ids = [str(a)+'.txt' for a in data_set_citations['publication_id'].values]\n",
    "publication_with_mentions_ids = set(publication_with_mentions_ids)\n",
    "for pub in publication_with_mentions_ids:\n",
    "    publication_with_mentions[pub] = full_text[pub]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average length of annotated papers:  6909.4828\n"
     ]
    }
   ],
   "source": [
    "print ('average length of annotated papers: ', np.mean([len(v.split()) for k, v in publication_with_mentions.items()]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22.245607551211112\n"
     ]
    }
   ],
   "source": [
    "# average sentence length \n",
    "t = [nltk.sent_tokenize(pub) for k, pub in publication_with_mentions.items()]\n",
    "print (np.mean([len(nltk.word_tokenize(s)) for pub in t for s in pub]))\n",
    "\n",
    "# we will use 40 as max sentence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.547755211308767\n"
     ]
    }
   ],
   "source": [
    "# average word length\n",
    "print (np.mean([len(w) for pub in t for s in pub for w in nltk.word_tokenize(s)])) \n",
    "\n",
    "# we will use 10 as max word length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "publication_dataset = defaultdict(list)\n",
    "publication_mention = defaultdict(list)\n",
    "for i in range(len(data_set_citations)):\n",
    "    row = data_set_citations.loc[i]\n",
    "    publication_dataset[row['publication_id']].append(row['data_set_id'])\n",
    "    publication_mention[row['publication_id']].extend(row['mention_list'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average number of datasets used in each paper:  2.1996\n"
     ]
    }
   ],
   "source": [
    "print ('average number of datasets used in each paper: ', np.mean([len(v) for k, v in publication_dataset.items()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average number of mentions per paper:  7.4952\n"
     ]
    }
   ],
   "source": [
    "print ('average number of mentions per paper: ', np.mean([len(v) for k, v in publication_mention.items()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean mention length:  4.663678087309211\n"
     ]
    }
   ],
   "source": [
    "print ('mean mention length: ', np.mean([len(m.split()) for k,v in publication_mention.items() for m in v]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In summary, we are using an annotated dataset of 2.5K papers. In this dataset, there are 2.2 datasets/paper and 7.5 mentions/paper on average."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}