process_huggingface.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process `thai_toxicity_tweet` for [huggingface/datasets](https://github.com/huggingface/datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import json\n",
    "from bs4 import BeautifulSoup\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "def get_tweet(tweet_id):\n",
    "    url = f'https://twitter.com/LllKnot/status/{str(tweet_id)}'\n",
    "    user_agent_old_phone = 'Nokia5310XpressMusic_CMCC/2.0 (10.10) Profile/MIDP-2.1 '\\\n",
    "    'Configuration/CLDC-1.1 UCWEB/2.0 (Java; U; MIDP-2.0; en-US; '\\\n",
    "    'Nokia5310XpressMusic) U2/1.0.0 UCBrowser/9.5.0.449 U2/1.0.0 Mobile'\n",
    "    headers = { 'User-Agent': user_agent_old_phone}\n",
    "    try:\n",
    "        with requests.get(url, headers=headers) as r:\n",
    "            soup = BeautifulSoup(r.text,'lxml')\n",
    "        return soup.find('div',class_='dir-ltr').text.strip()\n",
    "    except:\n",
    "        print(tweet_id)\n",
    "        return 'TWEET_NOT_FOUND'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('annotated_corpus.txt',sep='\\t',header=None)\n",
    "df.columns = ['is_toxic','votes','tweet_id']\n",
    "df['toxic_votes'] = df.votes.map(lambda x: x[1])\n",
    "df['nontoxic_votes'] = df.votes.map(lambda x: x[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "832751bd56634e66b3abb1faf92c0bea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "847331851425300481\n",
      "898003771048865794\n",
      "899572521388736512\n",
      "898774807386275840\n",
      "898921683242729473\n",
      "899553920560775170\n",
      "898752124183379968\n",
      "901097416786493440\n",
      "898327631581462528\n",
      "898327299468124161\n",
      "899441728876302336\n",
      "898224687129313280\n",
      "898590025767567361\n",
      "899247739153793025\n",
      "896878920313602048\n",
      "848208281436798976\n",
      "899310458707918848\n",
      "898204946176159744\n",
      "898221362950492160\n",
      "899612718742650880\n",
      "899608396000669697\n",
      "899312806071349248\n",
      "898556018354921472\n",
      "897789026437967872\n",
      "899301663055282179\n",
      "897667853326983168\n",
      "898592591125467136\n",
      "899300100123484161\n",
      "898622910985678848\n",
      "898580490260631552\n",
      "899178348437307393\n",
      "900299877409726464\n",
      "899821812602634240\n",
      "899951425408466949\n",
      "900239197067329540\n",
      "898465738242924544\n",
      "898550281453002752\n",
      "898580629834448896\n",
      "900346585946537984\n",
      "897530095245475840\n",
      "901462339710402560\n",
      "896821964634963968\n",
      "901460176615559168\n",
      "901458518485508096\n",
      "898953437517234176\n",
      "896422477936549888\n",
      "898912457661440001\n",
      "898616038874427392\n",
      "900365940386168832\n",
      "897491526648856576\n",
      "890477560865239044\n",
      "899279501493886977\n",
      "899278331081773056\n",
      "899523474141528064\n",
      "898551168363778048\n",
      "898234125332631552\n",
      "898237341365551104\n",
      "900364153407250432\n",
      "876277511407058944\n",
      "901007346851168257\n",
      "900941697269366784\n",
      "900795363346456577\n",
      "899261771902406656\n",
      "898207789197152256\n",
      "900590731265400832\n",
      "897127886842351616\n",
      "898019472715333632\n",
      "900859505155506176\n",
      "900902481676296193\n",
      "906635654582247424\n",
      "901091397918380032\n",
      "897501653657542656\n",
      "896800764177268736\n",
      "897011732433879040\n",
      "898380458991341569\n",
      "898736287846342656\n",
      "901080759900884993\n",
      "899447222500184064\n",
      "899962661902860288\n",
      "898920306412146689\n",
      "899451321786355712\n",
      "899565134007357440\n",
      "902549272742076416\n",
      "898605831016534016\n",
      "898221320147673088\n",
      "898555726792085504\n",
      "895393067020173313\n",
      "895335395881242625\n",
      "890248798013927425\n",
      "900007106023006208\n",
      "899256292845301762\n",
      "899544694388015105\n",
      "897619470373584900\n",
      "899572776503173120\n",
      "899796977927430145\n",
      "899551743410929664\n",
      "899403026657443840\n",
      "898566039750885376\n",
      "901147650078015488\n",
      "903128493701062656\n",
      "903224816324362243\n",
      "903224638490025984\n",
      "898604901294424064\n",
      "898795645703602176\n",
      "899268406603530240\n",
      "901112132942446592\n",
      "899584900445028354\n",
      "903251864610758659\n",
      "899570743997997056\n",
      "903246801892978688\n",
      "898715215591653377\n",
      "903245911932903424\n",
      "898790458372915201\n",
      "903120212588871680\n",
      "896056009982058497\n",
      "903228725533204480\n",
      "896481400982953984\n",
      "899246072559685632\n",
      "900833081476632576\n",
      "852347446785040384\n",
      "899486299316301824\n",
      "900693559094910980\n",
      "900668274077081600\n",
      "903158713783017472\n",
      "898162644703522817\n",
      "898488570628800512\n",
      "899140579438739456\n",
      "898559549514305536\n",
      "896775629630078976\n",
      "899225815719723008\n",
      "898587557578981376\n",
      "899255101595856896\n",
      "899253181284204544\n",
      "902851665774100481\n",
      "902927720396464129\n",
      "887919858578477057\n",
      "902849625874300928\n",
      "903085203878330369\n",
      "898762766059618304\n",
      "899588741337038856\n",
      "893076720525352960\n",
      "898608060247154689\n",
      "899268420591419392\n",
      "897132215070412801\n",
      "899773924610777089\n",
      "893057921147219968\n",
      "899579269008904193\n",
      "903576182293127168\n",
      "895334991642509313\n",
      "903576230267543552\n",
      "900338839461416967\n",
      "895416777877762050\n",
      "860528727695925248\n",
      "900200456529563649\n",
      "899674045695774721\n",
      "900030194534285313\n",
      "870321180149809152\n",
      "865426811571773442\n",
      "899492675463782401\n",
      "899589100432367617\n",
      "901140026976686080\n",
      "898504191701770240\n",
      "902949102194876416\n",
      "902550264946712576\n",
      "891221024292749312\n",
      "888466062047559680\n",
      "903825885442195456\n",
      "898548295408144385\n",
      "898177888880435201\n",
      "899567971860791296\n",
      "896490649247129604\n",
      "901207491240964096\n",
      "899263800133603328\n",
      "903921281535401984\n",
      "897500289858551808\n",
      "903771785849696256\n",
      "898210933146701824\n",
      "897588532901957632\n",
      "903773231286325248\n",
      "898183134595829760\n",
      "895623739433406464\n",
      "903784011671244801\n",
      "901230591009464320\n",
      "903476420399136769\n",
      "898945503320096769\n",
      "901072751833645056\n",
      "890132481646747649\n",
      "898943147891347456\n",
      "899114621721640966\n",
      "899477649034141696\n",
      "897851512528846848\n",
      "853137218277212160\n",
      "898920881950294020\n",
      "898464833602936834\n",
      "902575958233776128\n",
      "898911431218782210\n",
      "900801326199721984\n",
      "903570998997393408\n",
      "895919434832896001\n",
      "891184787360231429\n",
      "903393350929309698\n",
      "901998361686941700\n",
      "900286350271709185\n",
      "901045539654877184\n",
      "899827836839645184\n",
      "898477075912249345\n",
      "898876243528925184\n",
      "897503466490220544\n",
      "888310086455640064\n",
      "899175460273246208\n",
      "886538614611427328\n",
      "898896749338558465\n",
      "906183228112769024\n",
      "862585749836767234\n",
      "900910661542191108\n",
      "893669681671819264\n",
      "900541920526475264\n",
      "902550100496441344\n",
      "900848060422512640\n",
      "868853235192168448\n",
      "899976715899777028\n",
      "898607103702482944\n",
      "887523245653504000\n",
      "901823604580483072\n",
      "853034510966861824\n",
      "901091095349673984\n",
      "906747103166160896\n",
      "901106654107664385\n",
      "906069819190984705\n",
      "906720352440422400\n",
      "905429629460070400\n",
      "902410828380487680\n",
      "890893016759623680\n",
      "898245380898398208\n",
      "899253788732608512\n",
      "857069646347878400\n",
      "848866645657272320\n",
      "898140016399994881\n",
      "900360870043066368\n",
      "902390328618975232\n",
      "906357742775951360\n",
      "896812652508229632\n",
      "898607846073421824\n",
      "906350748358676480\n",
      "899251355579056128\n",
      "899550112061415424\n",
      "896635783569752064\n",
      "898933976689135617\n",
      "850421511005421568\n",
      "900346348473532419\n",
      "905113203784351744\n",
      "898560851996573698\n",
      "902946768236625920\n",
      "902727565034496000\n",
      "900735864560533505\n",
      "899561055034028032\n",
      "902468978273947648\n",
      "902446619475402756\n",
      "907847589567049728\n",
      "902865874423390212\n",
      "902405750030458880\n",
      "906538085877374976\n",
      "907763987139325952\n",
      "907840036871294976\n",
      "905218084155277312\n",
      "902792450497929216\n",
      "898357293003100162\n",
      "901114059352809472\n",
      "906782243233841153\n",
      "864148106513858560\n",
      "907199001543323649\n",
      "898428765004374016\n",
      "907288790674243584\n",
      "907562085160148993\n",
      "906723768957001728\n",
      "906212342781812737\n",
      "905579113661972480\n",
      "905387182520598528\n",
      "826471463242698752\n",
      "817269980555153408\n",
      "893426533703888896\n",
      "907957225657733120\n",
      "902873972982743040\n",
      "907952659331637248\n",
      "904372338438807553\n",
      "890808242518990849\n",
      "888287639580753920\n",
      "853185614698618880\n",
      "907887548059443200\n",
      "903198120430804992\n",
      "899605641664151553\n",
      "869936360336969728\n",
      "890423566943961088\n",
      "902866847141203968\n",
      "900525397342732289\n",
      "855394895116120064\n",
      "906907408114499584\n",
      "922830229789278208\n",
      "888299000314552320\n",
      "898230274718289920\n",
      "893342378831917056\n",
      "902193424111083520\n",
      "861475705573351425\n",
      "891137351518363648\n",
      "905395739743760384\n",
      "825620156185325568\n",
      "892427250284412928\n",
      "892052848564871168\n",
      "891683169207844864\n",
      "894756462295105538\n",
      "862280057925976065\n",
      "888534573537050625\n",
      "853952463300198400\n",
      "900888695678488576\n",
      "902538419720536064\n",
      "901088762595127296\n",
      "906548629099261952\n",
      "907576858065039360\n",
      "891523970700197888\n",
      "886174584407445504\n",
      "898921482293714944\n",
      "898947911773376512\n",
      "899572006152044544\n",
      "899749436649201665\n",
      "899844301978427394\n",
      "890501421270945792\n",
      "896616116562214912\n",
      "869274839386734592\n",
      "897868474692780032\n",
      "898221730245656576\n",
      "896805773338005504\n",
      "903151780149596160\n",
      "903873840018685954\n",
      "885718101219622912\n",
      "903862902838386688\n",
      "856830173403938816\n",
      "906879215185813504\n",
      "878622995358470144\n",
      "901798026829635584\n",
      "901390143222718464\n",
      "902462324971192321\n",
      "901833947763507201\n",
      "899516817521000448\n",
      "898568911351709696\n",
      "922830265197592576\n",
      "919542197501882369\n",
      "899387352451121152\n",
      "898949143506243585\n",
      "899574651457748992\n",
      "898793854802870272\n",
      "899357737653485568\n",
      "896449900564553728\n",
      "899576248053022720\n",
      "896471955250925568\n",
      "899567036270911488\n",
      "921955548785926150\n",
      "922027954720223233\n",
      "902898570780581889\n",
      "924954720883507201\n",
      "939130320363720705\n",
      "898220857113272320\n",
      "926552154759819265\n",
      "928672462249721856\n",
      "913172133265600514\n",
      "913509847408115712\n",
      "898741621566488577\n",
      "913124724598697984\n",
      "898763806804803584\n",
      "928853603757973504\n",
      "913073430085300224\n",
      "939314712780554240\n",
      "928820697425514496\n",
      "913249114342621189\n",
      "913442615302881280\n",
      "899605830986629120\n",
      "898826515583873024\n",
      "899564724186042369\n",
      "900155729109106688\n",
      "898548129795940352\n",
      "863379698604273668\n",
      "898833020198699009\n",
      "894171112040407040\n",
      "913005369206571009\n",
      "898914888965214208\n",
      "913022582349209601\n",
      "926432682560458752\n",
      "925722932982280192\n",
      "898937667819151364\n",
      "898950386794377216\n",
      "898925211579326464\n",
      "895335177261469696\n",
      "922065276362825728\n",
      "913116228050329600\n",
      "922091869399523328\n",
      "900345480529321985\n",
      "939187687579205632\n",
      "869538729953599488\n",
      "927742217405448192\n",
      "926359830868107264\n",
      "925653578982633473\n",
      "913024547720388608\n",
      "925789014669398017\n",
      "925791154209112064\n",
      "928280094736506880\n",
      "913006168259231744\n",
      "913023571147948032\n",
      "928586464908132355\n",
      "928600677474975745\n",
      "928620550817161216\n",
      "913073051620610050\n",
      "912923575949987840\n",
      "900351530959634433\n",
      "899654781324607488\n",
      "919161330757595142\n",
      "899600446142074881\n",
      "913161190213160960\n",
      "921624571911925760\n",
      "850416895773638656\n",
      "898237986856353792\n",
      "926623445676204032\n",
      "918693985249456128\n",
      "913072279914872832\n",
      "900355502151946240\n",
      "913123789390491649\n",
      "913139775938355200\n",
      "928190014889930752\n",
      "920979301628915712\n",
      "897098692083056640\n",
      "913120965114028032\n",
      "913529840703234048\n",
      "913537861990752257\n",
      "928926346536820736\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "928952784497524737\n",
      "928141947427889152\n",
      "896881805797736448\n",
      "913121839555141632\n",
      "899253477104209921\n",
      "899249402702843904\n",
      "897743022581964800\n",
      "897648590813843458\n",
      "899260651499012096\n",
      "913077445867200512\n",
      "913076063516868609\n",
      "913076656843010048\n",
      "913074764125634561\n",
      "913077024624803841\n",
      "924594417138221057\n",
      "896726030118813697\n",
      "913140736735850496\n",
      "897860210324152321\n",
      "928647453938552833\n",
      "901820363964801024\n",
      "928651216124395520\n",
      "922104394664964098\n",
      "928654447957360640\n",
      "928649151209816064\n",
      "922105830408781826\n",
      "899962782526734336\n",
      "899794420232470528\n",
      "847913056638193664\n",
      "896737321143238657\n",
      "899345390910164992\n",
      "919429391054127104\n",
      "899441395244515328\n",
      "899275957193687041\n",
      "899821013441036288\n",
      "898229285709688832\n",
      "898235739657216000\n",
      "898955969593356288\n",
      "913477904339628032\n",
      "903737056198873090\n",
      "898593557451227136\n",
      "928907673885478913\n",
      "938742479200796672\n",
      "937146543416664064\n",
      "913095311756636160\n",
      "928989539170885633\n",
      "928982422917541890\n",
      "928955154484695040\n",
      "928948524284178432\n",
      "928905754513367040\n",
      "927934249684017152\n",
      "927836719801495552\n",
      "928418726411583488\n",
      "898308488702459908\n",
      "913047169359265792\n",
      "898580670087217152\n",
      "913027068408897538\n",
      "898564235696852993\n",
      "939278644278575104\n",
      "939317192285233152\n",
      "928865023564251136\n",
      "928731015551508480\n",
      "898228247950876673\n",
      "913128482393038848\n",
      "913144390708629504\n",
      "898943734192062465\n",
      "913088006323527680\n",
      "925785627194302464\n",
      "927058947483111429\n",
      "897304525723258880\n",
      "900728539145658368\n",
      "925876820775333888\n",
      "928834872369561600\n",
      "925052754304811008\n",
      "901820864550772736\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tweet_texts = []\n",
    "for tweet_id in tqdm(df.tweet_id.tolist()):\n",
    "    tweet_texts.append(get_tweet(tweet_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['tweet_text'] = tweet_texts\n",
    "df = df[['tweet_id','tweet_text','toxic_votes','nontoxic_votes','is_toxic']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #save to jsonl\n",
    "# df.to_json('train.jsonl',orient='records',lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>tweet_text</th>\n",
       "      <th>toxic_votes</th>\n",
       "      <th>nontoxic_votes</th>\n",
       "      <th>is_toxic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>898576382384418817</td>\n",
       "      <td>วันๆ นี่คุยกะหมา แมว หมู ไก่ ม้า ควาย มากกว่าค...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>847331851425300481</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>899587505493692417</td>\n",
       "      <td>หล่อมากกก หล่อวัวตายควายล้มกันเลยทีเดียวววว</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>898920493763280897</td>\n",
       "      <td>สิวเหี้ยไรขึ้นหลังหู เสียชาติเกิดมั้ย เกิดมามั...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>896808964326694912</td>\n",
       "      <td>อ่ะ ป่วยก็ป่วย งานจ้างก็ต้องทำ งานเรียนก็ต้องท...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3295</th>\n",
       "      <td>913124142638964737</td>\n",
       "      <td>ควย การตอบว่าเสือกนี่แม่งไม่ตรงประเด็นเหี้ยๆ</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3296</th>\n",
       "      <td>900296425937846272</td>\n",
       "      <td>ส้นตีนมากค่ะ อาเซียนร่วมใจจริงๆ -_-</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3297</th>\n",
       "      <td>912997420035842048</td>\n",
       "      <td>ไม่สวยก็เหนื่อยหน่อย ไม่หน่อยหรอกอีผี เหนื่อยฉ...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3298</th>\n",
       "      <td>903577626756792320</td>\n",
       "      <td>นี่คือประเทศไทยในความจริง คนชั่วไม่ต้องติดคุกแ...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3299</th>\n",
       "      <td>913551785830125569</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3300 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                         tweet_text  \\\n",
       "0     898576382384418817  วันๆ นี่คุยกะหมา แมว หมู ไก่ ม้า ควาย มากกว่าค...   \n",
       "1     847331851425300481                                    TWEET_NOT_FOUND   \n",
       "2     899587505493692417        หล่อมากกก หล่อวัวตายควายล้มกันเลยทีเดียวววว   \n",
       "3     898920493763280897  สิวเหี้ยไรขึ้นหลังหู เสียชาติเกิดมั้ย เกิดมามั...   \n",
       "4     896808964326694912  อ่ะ ป่วยก็ป่วย งานจ้างก็ต้องทำ งานเรียนก็ต้องท...   \n",
       "...                  ...                                                ...   \n",
       "3295  913124142638964737       ควย การตอบว่าเสือกนี่แม่งไม่ตรงประเด็นเหี้ยๆ   \n",
       "3296  900296425937846272                ส้นตีนมากค่ะ อาเซียนร่วมใจจริงๆ -_-   \n",
       "3297  912997420035842048  ไม่สวยก็เหนื่อยหน่อย ไม่หน่อยหรอกอีผี เหนื่อยฉ...   \n",
       "3298  903577626756792320  นี่คือประเทศไทยในความจริง คนชั่วไม่ต้องติดคุกแ...   \n",
       "3299  913551785830125569                                                      \n",
       "\n",
       "     toxic_votes nontoxic_votes  is_toxic  \n",
       "0              0              3         0  \n",
       "1              3              0         1  \n",
       "2              0              3         0  \n",
       "3              0              3         0  \n",
       "4              0              3         0  \n",
       "...          ...            ...       ...  \n",
       "3295           0              3         0  \n",
       "3296           3              0         1  \n",
       "3297           0              3         0  \n",
       "3298           0              3         0  \n",
       "3299           3              0         1  \n",
       "\n",
       "[3300 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs = []\n",
    "with open('train.jsonl','r') as f:\n",
    "    for line in f:\n",
    "        dfs.append(json.loads(line))\n",
    "df = pd.DataFrame(dfs)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>tweet_text</th>\n",
       "      <th>toxic_votes</th>\n",
       "      <th>nontoxic_votes</th>\n",
       "      <th>is_toxic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>847331851425300481</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>898003771048865794</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>899572521388736512</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>898774807386275840</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>898921683242729473</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3250</th>\n",
       "      <td>900728539145658368</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3259</th>\n",
       "      <td>925876820775333888</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3260</th>\n",
       "      <td>928834872369561600</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3267</th>\n",
       "      <td>925052754304811008</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3291</th>\n",
       "      <td>901820864550772736</td>\n",
       "      <td>TWEET_NOT_FOUND</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>506 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id       tweet_text toxic_votes nontoxic_votes  is_toxic\n",
       "1     847331851425300481  TWEET_NOT_FOUND           3              0         1\n",
       "18    898003771048865794  TWEET_NOT_FOUND           3              0         1\n",
       "38    899572521388736512  TWEET_NOT_FOUND           3              0         1\n",
       "42    898774807386275840  TWEET_NOT_FOUND           3              0         1\n",
       "49    898921683242729473  TWEET_NOT_FOUND           2              1         1\n",
       "...                  ...              ...         ...            ...       ...\n",
       "3250  900728539145658368  TWEET_NOT_FOUND           3              0         1\n",
       "3259  925876820775333888  TWEET_NOT_FOUND           0              3         0\n",
       "3260  928834872369561600  TWEET_NOT_FOUND           3              0         1\n",
       "3267  925052754304811008  TWEET_NOT_FOUND           3              0         1\n",
       "3291  901820864550772736  TWEET_NOT_FOUND           0              3         0\n",
       "\n",
       "[506 rows x 5 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.tweet_text=='TWEET_NOT_FOUND']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}