seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.1.0\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "import time\n",
    "from tensorflow.python.layers.core import Dense\n",
    "from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83568\n",
      "83568\n"
     ]
    }
   ],
   "source": [
    "path = \"cnn_dailymail/cnn/training\" #文件夹目录  \n",
    "files= os.listdir(path) #得到文件夹下的所有文件名称 \n",
    "count = 0\n",
    "all_article = [] \n",
    "all_target = []\n",
    "for file in files: #遍历文件夹  \n",
    "#     count +=1\n",
    "#     if count >= 10000:\n",
    "#         break;\n",
    "    if not os.path.isdir(file): #判断是否是文件夹，不是文件夹才打开  \n",
    "        f = open(path+\"/\"+file); #打开文件  \n",
    "        iter_f = iter(f); #创建迭代器  \n",
    "        tab = ' '\n",
    "        lines = []\n",
    "        target = []\n",
    "        n_count = 0\n",
    "        for line in iter_f: #遍历文件，一行行遍历，读取文本  \n",
    "            if line.strip()!='' and line.find('\\t0')==-1 and line.find('\\t\\t\\t')!=-1: # 去掉标记为0的sentence\n",
    "                lines.append(line.strip()[:-4])\n",
    "            elif line.find('\\t\\t\\t')==-1:\n",
    "                if line == '\\n':\n",
    "                    n_count+=1\n",
    "                else:\n",
    "                    if n_count >= 3:\n",
    "                        break;\n",
    "                    target.append(line.strip())\n",
    "        lines = tab.join(lines)\n",
    "        target = target[1:]\n",
    "        target = tab.join(target)\n",
    "        all_article.append(lines) #每个文件的文本存到list中  \n",
    "        all_target.append(target)\n",
    "        \n",
    "print(len(all_article)) \n",
    "print(len(all_target))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reviews = pd.DataFrame({'Summary':all_target,'Text':all_article})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review # 1\n",
      "@entity11 has called for anyone with information to share it according to local media , children in @entity32 stumbled upon the grave in the 1970s a local historian has concluded that children from a @entity2 - run home were buried there\n",
      "-- the revelation that nearly 800 children may have been buried in an unmarked mass grave at a former @entity2 - run home for unwed mothers in @entity9 is \" sickening \" and must be investigated , the country 's top @entity2 clergyman says @entity11 @entity10 has issued a statement urging anyone with information about mass graves to go to the authorities \" @entity19 message is that authentic faith is measured by how we treat children who represent @entity23 , \" @entity10 said in the june 5 statement on june 1 , the @entity24 cited a local historian as saying she believed 796 children who had died between 1925 and 1961 at a home run by the @entity28 had been buried in an unofficial grave beside the home 's old location in @entity32 , @entity33 local children stumbled upon the grave in the 1970s , @entity9 media reported , but the site was never examined afterward the historian , @entity38 , told @entity39 saturday that she had bought death certificates for the children but she found that there had been no burial records the revelation has sparked calls for an investigation and renewed questions about the treatment of unmarried mothers and their children by the @entity51 and institutions associated with it ' full bodied inquiry ' the @entity54 said @entity10 had asked the @entity56 archivist to compile information on mother and baby homes in @entity60 several months ago hundreds of documents had been collated but none so far contained information concerning mass graves , it said \" the archbishop has said he will share this information with any inquiry the government will establish the archbishop called for a commission to be established to study the social context of the period government response minister for children and youth affairs @entity102 said in a statement wednesday that \" active consideration \" was being given to how to address the details that had emerged \" many of the revelations are deeply disturbing and a shocking reminder of a darker past in @entity9 when our children were not cherished as they should have been , \" he said archbishop @entity121 , who heads the @entity32 archdiocese , last week welcomed any government move to examine what happened at the home \" it will be a priority for me , in cooperation with the families of the deceased , to seek to obtain a dignified re-interment of the remains of the children in consecrated ground in @entity32 , \" he said ' @entity134 ' the @entity32 case is the latest high - profile episode in which the state and @entity51 have been called to account over care of the most vulnerable in @entity9 society a government report last year into the so - called @entity141 , run by various @entity2 orders , acknowledged that @entity9 's government sent thousands of women and girls to \" harsh and physically demanding \" workhouses , where they worked and lived without pay , sometimes for years\n",
      "\n",
      "Review # 2\n",
      "up to 3,200 passengers to be taken across @entity12 on seven boats service is partially a response to the closure of all bridges from @entity7 to @entity0 @entity12 clears security to provide service all passengers must arrive at least an hour early for a full screening\n",
      "@entity0 -- while nearly all maritime activity on the @entity5 will be halted on @entity6 , a handful of mall - goers will sail from @entity7 to @entity0 on water taxis working closely with the @entity11 , the @entity12 , based in @entity13 , @entity7 , plans to ferry up to 3,200 passengers from across the frigid @entity12 on seven climate - controlled boats the service is partially a response to the closure of all bridges from @entity7 to @entity0 on @entity6 , a measure that prompted an uproar among @entity7 residents according to maritime authorities in charge of the region , @entity12 is the only company being allowed to taxi across the river on the 20th \" because the taxi service provided us with a security plan that we examined and deemed adequate to provide good security , i granted them a waiver to operate at an increased security level , \" @entity30 port captain @entity29 says \" the @entity11 established a temporary security zone on waterways in the @entity30 , stepping up restrictions day - by - day leading up to 11 p.m. on january 19 \" the boats depart the @entity62 for the 40 - minute trip beginning around 6 a.m. and must be docked on the southwest @entity0 shore by 10 a.m. from there , it 's a cold , mile - long walk to the @entity67 , so @entity68 has some advice for passengers \" show up early to get through the security check and dress warm , \" he said \" return trips , tentatively scheduled for around 6 p.m. , are not allowed to depart @entity0 until after the inaugural parade events have concluded and president @entity77 is secured in keeping with the @entity11 's security measures , @entity48 says all passengers must arrive at least an hour early for a full screening passengers that want to stay in @entity0 after the early evening departures will have to find other transport back across the @entity12 @entity12 vice president @entity85 says tickets went on sale monday and the initial response from customers has been enthusiastic .\n",
      "\n",
      "Review # 3\n",
      "@entity205 *laments* @entity13 's getting bumped from a flight because of his size she says such treatment is routine for fliers now , *unlike* the days when flying was fun @entity215 -- security measures , no food -- begin early and last till you *'ve* landed , she says @entity205 : it 's only a matter of time till @entity13 's treatment becomes routine even off the plane\n",
      "@entity0 , @entity1 -- as many of you already know -- because tweets travel faster than airplanes -- @entity13 , the portly film director of the movie \" @entity15 \" and the upcoming film \" @entity12 , \" was \" politely \" asked to disembark from his @entity7 flight from @entity9 to @entity10 because he was too large to fit in the seat okay , let 's be frank : he was kicked off the flight the reason he was given was that his size infringed upon the space of the person sitting next to him and it could cause a safety risk in case of an emergency does it surprise you that an airline that has charged a customer several hundred dollars for the service of transporting him from one place to another would subject said paying customer to the indignity of being told he 's too fat to fly ? i do n't think it does because this is what flying has become : barely one rung above bus travel @entity7 may have apologized to @entity13 , but what about the rest of us ? scratch the surface of any frequent flier and you will find a raging cauldron of resentment ( or endless angry tweets , like @entity57 ) , a bottomless pit of despair and rage with no hope of satisfaction because , after all , fliers have no choice but to accept every humiliation if they want to fly most of us do n't have our own planes -- or fly business class myself , i 'm old enough to remember when flying used to be fun -- but not quite old enough to remember when people used to \" dress \" to fly the tragedy of 9/11 hastened the end of any kind of special treatment boarding became a cattle call -- complete with prodding with a stick -- where you had to basically undress before you could get on the plane , take off shoes , coats , sweaters , get \" wanded , \" step out of the line for frisking , and other debasements too horrible to mention really ! i know one woman whose underwire bra had security stumped ) by the time you get on the plane , you are like a refugee -- the tired , the poor , the huddled masses in middle seats flight attendants ca n't dote on you any more , they are too busy scanning the group for potential terrorists no more meals -- bad as they were -- no more cutlery it makes you long for the fat days ( you 'll pardon the expression ) of @entity135 , that 1980 's no - frills airline that operated out of @entity137 it seemed pretty terrible back then , but really it was just a preview -- every man for himself basically anything you got during the flight , you paid extra for now i 'm wondering if that 's a good enough reason anymore how much can a body take , even a skinny one like mine ? @entity13 , i feel your pain ! i cringe for the person who has to sheepishly ask for the seat belt extender because the one in the narrow little seat does n't make it around his body the world is not built for fat people , and airplanes are not the place to look for gentle treatment i will not be surprised when airlines start to weigh people 's luggage at the ticket counter , and then ask the traveler to get on the scale and be charged by the pound it 's coming , you watch if a fat person has to pay extra to fly -- @entity13 usually buys two seats on @entity7 because they do n't have first class -- then this could be a precedent for more suffering in the world away from the tarmac a fat person will have to pay extra for a larger size in clothing the opinions expressed in this commentary are solely those of @entity205\n",
      "\n",
      "Review # 4\n",
      "@entity8 says he 's under a \" death sentence \" \" there 's nothing greater than for a family member to receive an organ \" doctor : physicians are unlikely to go along with the @entity1 man 's plan some @entity65 sufferers live for decades with the disease\n",
      "@entity0 , @entity1 -- a @entity1 man suffering from @entity5 's disease says he wants to die by having his organs harvested rather than wait for his degenerative nerve ailment to kill him @entity8 has been battling amyotrophic lateral sclerosis , as the disease is formally known , since 2008 the 61 - year - old told @entity11 's \" @entity14 \" on thursday that the diagnosis is \" a death sentence , positively , no questions asked , \" and he wants do donate his organs while he still can \" if people have their legs cut off or their wife left them and they commit suicide , it 's another story and for people waiting for transplants , \" there 's nothing greater than for a family member to receive an organ so they can watch their family grow up \" @entity5 's disease attacks the nerves in the brain and spinal cord , affecting voluntary muscle control in its later stages , the disease can leave sufferers paralyzed but fully conscious and alert @entity8 lives in @entity40 , @entity1 , about 50 miles north of @entity0 but she said that when her father 's disease \" gets to the point where he 's lost his pride or become humiliated in how he 's having to suffer and the indignities ... we 're not going to have an issue with that \" about 5,600 @entity63 a year are diagnosed with the disease , according to the @entity65 , a nonprofit group that advocates for more research on the disease @entity8 ' offer is \" admirable , and our heart goes out to him , \" @entity71 , an emergency medicine physician at the @entity72 , told @entity11 \" but 10 percent of people can live 20 years with this disease \" @entity71 said what @entity8 wants to do amounts to physician - assisted suicide , a practice legal in only three @entity63 states -- @entity84 , @entity85 and @entity86 most doctors \" would have a problem \" with what @entity8 wants to do , because \" at this point , he 's fairly healthy and @entity65 is unlikely to harm the organs most likely to be used for transplants unless some sort of related infection strikes them , she said @entity8 has also taken his appeal online\n",
      "\n",
      "Review # 5\n",
      "@entity131 address was @entity45 's on january 22 , 1946 the shortest : @entity64 's 1790 address former president @entity16 was a big *fan* of wearing blue ties\n",
      "among the notable examples are @entity27 in 1990 and 1994 and @entity29 president @entity28 in 2006 -- the first annual message broadcast on radio was president @entity44 's speech on december 6 , 1923 -- the first televised @entity34 was delivered by president @entity45 on january 6 , 1947 -- the first prime - time @entity34 was delivered on january 4 , 1965 president @entity47 moved the speech from its traditional midday time slot to the evening to attract a larger television audience -- the first @entity34 to be webcast live on the internet was president @entity16 's 2002 address president @entity54 was scheduled to deliver his address on january 28 , 1986 , the same day the space shuttle @entity57 exploded -- @entity59 : @entity45 's first @entity34 address , on january 22 , 1946 , was more than 25,000 words -- the shortest : @entity64 's 1790 address was only 833 words and is believed to have lasted less than 10 minutes just the facts , ma'am @entity13 : -- wednesday , january 27 , 2010 : 69 minutes long and 89 applause interruptions @entity16 : -- monday , january 28 , 2008 : 53 minutes and 70 applause interruptions -- wednesday , january 23 , 2007 : 50 minutes and 63 applause interruptions -- tuesday , january 31 , 2006 : 51 minutes and 64 applause interruptions -- tuesday , january 20 , 2004 : 54 minutes and 67 applause interruptions -- tuesday , january 28 , 2003 : 67 minutes including applause time ; 19 \" @entity81 \" references ; 22 \" terror \" / \" terrorism \" / \" terrorist \" references ; one seat left empty in the gallery in honor of september 11 victims -- tuesday , january 29 , 2002 : 48 minutes and 78 applause interruptions @entity17 : -- thursday , january 27 , 2000 : 88 minutes , @entity17 's longest and 128 applause interruptions -- a record for him -- tuesday , january 19 , 1999 : 77 minutes and 95 applause interruptions -- tuesday , january 27 , 1998 : 72 minutes ; length of applause was four minutes during the speech ; the welcoming applause lasted 2 minutes , 25 seconds ; 101 applause interruptions -- tuesday , january 23 , 1996 : 62 minutes ; more than 60 applause interruptions -- tuesday , january 24 , 1995 : 81 minutes ; more than 96 bursts of applause ; 39 standing ovations -- tuesday , january 25 , 1994 : 63 minutes ; 60 rounds of applause and 18 standing ovations @entity123 -- tuesday , january 28 , 1992 : 50 minutes and 76 applause interruptions -- tuesday , january 29 , 1991 : 47 minutes and more than 50 bursts of applause\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    print(\"Review #\",i+1)\n",
    "    print(reviews.Summary[i])\n",
    "    print(reviews.Text[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A list of contractions from http://stackoverflow.com/questions/19790188/expanding-english-language-contractions-in-python\n",
    "contractions = { \n",
    "\"ain't\": \"am not\",\n",
    "\"aren't\": \"are not\",\n",
    "\"can't\": \"cannot\",\n",
    "\"can't've\": \"cannot have\",\n",
    "\"'cause\": \"because\",\n",
    "\"could've\": \"could have\",\n",
    "\"couldn't\": \"could not\",\n",
    "\"couldn't've\": \"could not have\",\n",
    "\"didn't\": \"did not\",\n",
    "\"doesn't\": \"does not\",\n",
    "\"don't\": \"do not\",\n",
    "\"hadn't\": \"had not\",\n",
    "\"hadn't've\": \"had not have\",\n",
    "\"hasn't\": \"has not\",\n",
    "\"haven't\": \"have not\",\n",
    "\"he'd\": \"he would\",\n",
    "\"he'd've\": \"he would have\",\n",
    "\"he'll\": \"he will\",\n",
    "\"he's\": \"he is\",\n",
    "\"how'd\": \"how did\",\n",
    "\"how'll\": \"how will\",\n",
    "\"how's\": \"how is\",\n",
    "\"i'd\": \"i would\",\n",
    "\"i'll\": \"i will\",\n",
    "\"i'm\": \"i am\",\n",
    "\"i've\": \"i have\",\n",
    "\"isn't\": \"is not\",\n",
    "\"it'd\": \"it would\",\n",
    "\"it'll\": \"it will\",\n",
    "\"it's\": \"it is\",\n",
    "\"let's\": \"let us\",\n",
    "\"ma'am\": \"madam\",\n",
    "\"mayn't\": \"may not\",\n",
    "\"might've\": \"might have\",\n",
    "\"mightn't\": \"might not\",\n",
    "\"must've\": \"must have\",\n",
    "\"mustn't\": \"must not\",\n",
    "\"needn't\": \"need not\",\n",
    "\"oughtn't\": \"ought not\",\n",
    "\"shan't\": \"shall not\",\n",
    "\"sha'n't\": \"shall not\",\n",
    "\"she'd\": \"she would\",\n",
    "\"she'll\": \"she will\",\n",
    "\"she's\": \"she is\",\n",
    "\"should've\": \"should have\",\n",
    "\"shouldn't\": \"should not\",\n",
    "\"that'd\": \"that would\",\n",
    "\"that's\": \"that is\",\n",
    "\"there'd\": \"there had\",\n",
    "\"there's\": \"there is\",\n",
    "\"they'd\": \"they would\",\n",
    "\"they'll\": \"they will\",\n",
    "\"they're\": \"they are\",\n",
    "\"they've\": \"they have\",\n",
    "\"wasn't\": \"was not\",\n",
    "\"we'd\": \"we would\",\n",
    "\"we'll\": \"we will\",\n",
    "\"we're\": \"we are\",\n",
    "\"we've\": \"we have\",\n",
    "\"weren't\": \"were not\",\n",
    "\"what'll\": \"what will\",\n",
    "\"what're\": \"what are\",\n",
    "\"what's\": \"what is\",\n",
    "\"what've\": \"what have\",\n",
    "\"where'd\": \"where did\",\n",
    "\"where's\": \"where is\",\n",
    "\"who'll\": \"who will\",\n",
    "\"who's\": \"who is\",\n",
    "\"won't\": \"will not\",\n",
    "\"wouldn't\": \"would not\",\n",
    "\"you'd\": \"you would\",\n",
    "\"you'll\": \"you will\",\n",
    "\"you're\": \"you are\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_text(text, remove_stopwords = True):\n",
    "    '''Remove unwanted characters, stopwords, and format the text to create fewer nulls word embeddings'''\n",
    "    \n",
    "    # Convert words to lower case\n",
    "    text = text.lower()\n",
    "    \n",
    "    # Replace contractions with their longer forms \n",
    "    if True:\n",
    "        text = text.split()\n",
    "        new_text = []\n",
    "        for word in text:\n",
    "            if word in contractions:\n",
    "                new_text.append(contractions[word])\n",
    "            else:\n",
    "                new_text.append(word)\n",
    "        text = \" \".join(new_text)\n",
    "    \n",
    "    # Format words and remove unwanted characters\n",
    "    text = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', text, flags=re.MULTILINE)\n",
    "    text = re.sub(r'\\<a href', ' ', text)\n",
    "    text = re.sub(r'&amp;', '', text) \n",
    "    text = re.sub(r'[_\"\\-;%()|+&=*%.,!?:#$@\\[\\]/]', ' ', text)\n",
    "    text = re.sub(r'<br />', ' ', text)\n",
    "    text = re.sub(r'\\'', ' ', text)\n",
    "    \n",
    "    # Optionally, remove stop words\n",
    "    if remove_stopwords:\n",
    "        text = text.split()\n",
    "        stops = set(stopwords.words(\"english\"))\n",
    "        text = [w for w in text if not w in stops]\n",
    "        text = \" \".join(text)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summaries are complete.\n",
      "Texts are complete.\n"
     ]
    }
   ],
   "source": [
    "# Clean the summaries and texts\n",
    "clean_summaries = []\n",
    "for summary in reviews.Summary:\n",
    "    clean_summaries.append(clean_text(summary, remove_stopwords=False))\n",
    "print(\"Summaries are complete.\")\n",
    "\n",
    "clean_texts = []\n",
    "for text in reviews.Text:\n",
    "    clean_texts.append(clean_text(text))\n",
    "print(\"Texts are complete.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clean Review # 1\n",
      " entity11 has called for anyone with information to share it according to local media   children in  entity32 stumbled upon the grave in the 1970s a local historian has concluded that children from a  entity2   run home were buried there\n",
      "revelation nearly 800 children may buried unmarked mass grave former entity2 run home unwed mothers entity9 sickening must investigated country top entity2 clergyman says entity11 entity10 issued statement urging anyone information mass graves go authorities entity19 message authentic faith measured treat children represent entity23 entity10 said june 5 statement june 1 entity24 cited local historian saying believed 796 children died 1925 1961 home run entity28 buried unofficial grave beside home old location entity32 entity33 local children stumbled upon grave 1970s entity9 media reported site never examined afterward historian entity38 told entity39 saturday bought death certificates children found burial records revelation sparked calls investigation renewed questions treatment unmarried mothers children entity51 institutions associated full bodied inquiry entity54 said entity10 asked entity56 archivist compile information mother baby homes entity60 several months ago hundreds documents collated none far contained information concerning mass graves said archbishop said share information inquiry government establish archbishop called commission established study social context period government response minister children youth affairs entity102 said statement wednesday active consideration given address details emerged many revelations deeply disturbing shocking reminder darker past entity9 children cherished said archbishop entity121 heads entity32 archdiocese last week welcomed government move examine happened home priority cooperation families deceased seek obtain dignified interment remains children consecrated ground entity32 said entity134 entity32 case latest high profile episode state entity51 called account care vulnerable entity9 society government report last year called entity141 run various entity2 orders acknowledged entity9 government sent thousands women girls harsh physically demanding workhouses worked lived without pay sometimes years\n",
      "\n",
      "Clean Review # 2\n",
      "up to 3 200 passengers to be taken across  entity12 on seven boats service is partially a response to the closure of all bridges from  entity7 to  entity0  entity12 clears security to provide service all passengers must arrive at least an hour early for a full screening\n",
      "entity0 nearly maritime activity entity5 halted entity6 handful mall goers sail entity7 entity0 water taxis working closely entity11 entity12 based entity13 entity7 plans ferry 3 200 passengers across frigid entity12 seven climate controlled boats service partially response closure bridges entity7 entity0 entity6 measure prompted uproar among entity7 residents according maritime authorities charge region entity12 company allowed taxi across river 20th taxi service provided us security plan examined deemed adequate provide good security granted waiver operate increased security level entity30 port captain entity29 says entity11 established temporary security zone waterways entity30 stepping restrictions day day leading 11 p january 19 boats depart entity62 40 minute trip beginning around 6 must docked southwest entity0 shore 10 cold mile long walk entity67 entity68 advice passengers show early get security check dress warm said return trips tentatively scheduled around 6 p allowed depart entity0 inaugural parade events concluded president entity77 secured keeping entity11 security measures entity48 says passengers must arrive least hour early full screening passengers want stay entity0 early evening departures find transport back across entity12 entity12 vice president entity85 says tickets went sale monday initial response customers enthusiastic\n",
      "\n",
      "Clean Review # 3\n",
      " entity205  laments   entity13  s getting bumped from a flight because of his size she says such treatment is routine for fliers now    unlike  the days when flying was fun  entity215    security measures   no food    begin early and last till you   ve  landed   she says  entity205   it  s only a matter of time till  entity13  s treatment becomes routine even off the plane\n",
      "entity0 entity1 many already know tweets travel faster airplanes entity13 portly film director movie entity15 upcoming film entity12 politely asked disembark entity7 flight entity9 entity10 large fit seat okay let frank kicked flight reason given size infringed upon space person sitting next could cause safety risk case emergency surprise airline charged customer several hundred dollars service transporting one place another would subject said paying customer indignity told fat fly n think flying become barely one rung bus travel entity7 may apologized entity13 rest us scratch surface frequent flier find raging cauldron resentment endless angry tweets like entity57 bottomless pit despair rage hope satisfaction fliers choice accept every humiliation want fly us n planes fly business class old enough remember flying used fun quite old enough remember people used dress fly tragedy 9 11 hastened end kind special treatment boarding became cattle call complete prodding stick basically undress could get plane take shoes coats sweaters get wanded step line frisking debasements horrible mention really know one woman whose underwire bra security stumped time get plane like refugee tired poor huddled masses middle seats flight attendants ca n dote busy scanning group potential terrorists meals bad cutlery makes long fat days pardon expression entity135 1980 frills airline operated entity137 seemed pretty terrible back really preview every man basically anything got flight paid extra wondering good enough reason anymore much body take even skinny one like mine entity13 feel pain cringe person sheepishly ask seat belt extender one narrow little seat n make around body world built fat people airplanes place look gentle treatment surprised airlines start weigh people luggage ticket counter ask traveler get scale charged pound coming watch fat person pay extra fly entity13 usually buys two seats entity7 n first class could precedent suffering world away tarmac fat person pay extra larger size clothing opinions expressed commentary solely entity205\n",
      "\n",
      "Clean Review # 4\n",
      " entity8 says he  s under a   death sentence     there  s nothing greater than for a family member to receive an organ   doctor   physicians are unlikely to go along with the  entity1 man  s plan some  entity65 sufferers live for decades with the disease\n",
      "entity0 entity1 entity1 man suffering entity5 disease says wants die organs harvested rather wait degenerative nerve ailment kill entity8 battling amyotrophic lateral sclerosis disease formally known since 2008 61 year old told entity11 entity14 thursday diagnosis death sentence positively questions asked wants donate organs still people legs cut wife left commit suicide another story people waiting transplants nothing greater family member receive organ watch family grow entity5 disease attacks nerves brain spinal cord affecting voluntary muscle control later stages disease leave sufferers paralyzed fully conscious alert entity8 lives entity40 entity1 50 miles north entity0 said father disease gets point lost pride become humiliated suffer indignities going issue 5 600 entity63 year diagnosed disease according entity65 nonprofit group advocates research disease entity8 offer admirable heart goes entity71 emergency medicine physician entity72 told entity11 10 percent people live 20 years disease entity71 said entity8 wants amounts physician assisted suicide practice legal three entity63 states entity84 entity85 entity86 doctors would problem entity8 wants point fairly healthy entity65 unlikely harm organs likely used transplants unless sort related infection strikes said entity8 also taken appeal online\n",
      "\n",
      "Clean Review # 5\n",
      " entity131 address was  entity45  s on january 22   1946 the shortest    entity64  s 1790 address former president  entity16 was a big  fan  of wearing blue ties\n",
      "among notable examples entity27 1990 1994 entity29 president entity28 2006 first annual message broadcast radio president entity44 speech december 6 1923 first televised entity34 delivered president entity45 january 6 1947 first prime time entity34 delivered january 4 1965 president entity47 moved speech traditional midday time slot evening attract larger television audience first entity34 webcast live internet president entity16 2002 address president entity54 scheduled deliver address january 28 1986 day space shuttle entity57 exploded entity59 entity45 first entity34 address january 22 1946 25 000 words shortest entity64 1790 address 833 words believed lasted less 10 minutes facts madam entity13 wednesday january 27 2010 69 minutes long 89 applause interruptions entity16 monday january 28 2008 53 minutes 70 applause interruptions wednesday january 23 2007 50 minutes 63 applause interruptions tuesday january 31 2006 51 minutes 64 applause interruptions tuesday january 20 2004 54 minutes 67 applause interruptions tuesday january 28 2003 67 minutes including applause time 19 entity81 references 22 terror terrorism terrorist references one seat left empty gallery honor september 11 victims tuesday january 29 2002 48 minutes 78 applause interruptions entity17 thursday january 27 2000 88 minutes entity17 longest 128 applause interruptions record tuesday january 19 1999 77 minutes 95 applause interruptions tuesday january 27 1998 72 minutes length applause four minutes speech welcoming applause lasted 2 minutes 25 seconds 101 applause interruptions tuesday january 23 1996 62 minutes 60 applause interruptions tuesday january 24 1995 81 minutes 96 bursts applause 39 standing ovations tuesday january 25 1994 63 minutes 60 rounds applause 18 standing ovations entity123 tuesday january 28 1992 50 minutes 76 applause interruptions tuesday january 29 1991 47 minutes 50 bursts applause\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Inspect the cleaned summaries and texts to ensure they have been cleaned well\n",
    "for i in range(5):\n",
    "    print(\"Clean Review #\",i+1)\n",
    "    print(clean_summaries[i])\n",
    "    print(clean_texts[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count_words(count_dict, text):\n",
    "    '''Count the number of occurrences of each word in a set of text'''\n",
    "    for sentence in text:\n",
    "        for word in sentence.split():\n",
    "            if word not in count_dict:\n",
    "                count_dict[word] = 1\n",
    "            else:\n",
    "                count_dict[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of Vocabulary: 81592\n"
     ]
    }
   ],
   "source": [
    "# Find the number of times each word was used and the size of the vocabulary\n",
    "word_counts = {}\n",
    "\n",
    "count_words(word_counts, clean_summaries)\n",
    "count_words(word_counts, clean_texts)\n",
    "            \n",
    "print(\"Size of Vocabulary:\", len(word_counts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word embeddings: 417195\n"
     ]
    }
   ],
   "source": [
    "# Load Conceptnet Numberbatch's (CN) embeddings, similar to GloVe, but probably better \n",
    "# (https://github.com/commonsense/conceptnet-numberbatch)\n",
    "embeddings_index = {}\n",
    "with open('/Users/manan/课程/nlp/numberbatch-en-17.06.txt', encoding='utf-8') as f:\n",
    "    for line in f:\n",
    "        values = line.split(' ')\n",
    "        word = values[0]\n",
    "        embedding = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = embedding\n",
    "\n",
    "print('Word embeddings:', len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words missing from CN: 1565\n",
      "Percent of words that are missing from vocabulary: 1.92%\n"
     ]
    }
   ],
   "source": [
    "# Find the number of words that are missing from CN, and are used more than our threshold.\n",
    "missing_words = 0\n",
    "threshold = 20\n",
    "\n",
    "for word, count in word_counts.items():\n",
    "    if count > threshold:\n",
    "        if word not in embeddings_index:\n",
    "            missing_words += 1\n",
    "            \n",
    "missing_ratio = round(missing_words/len(word_counts),4)*100\n",
    "            \n",
    "print(\"Number of words missing from CN:\", missing_words)\n",
    "print(\"Percent of words that are missing from vocabulary: {}%\".format(missing_ratio))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of unique words: 81592\n",
      "Number of words we will use: 63003\n",
      "Percent of words we will use: 77.22%\n"
     ]
    }
   ],
   "source": [
    "# Limit the vocab that we will use to words that appear ≥ threshold or are in GloVe\n",
    "\n",
    "#dictionary to convert words to integers\n",
    "vocab_to_int = {} \n",
    "\n",
    "value = 0\n",
    "for word, count in word_counts.items():\n",
    "    if count >= threshold or word in embeddings_index:\n",
    "        vocab_to_int[word] = value\n",
    "        value += 1\n",
    "\n",
    "# Special tokens that will be added to our vocab\n",
    "codes = [\"<UNK>\",\"<PAD>\",\"<EOS>\",\"<GO>\"]   \n",
    "\n",
    "# Add codes to vocab\n",
    "for code in codes:\n",
    "    vocab_to_int[code] = len(vocab_to_int)\n",
    "\n",
    "# Dictionary to convert integers to words\n",
    "int_to_vocab = {}\n",
    "for word, value in vocab_to_int.items():\n",
    "    int_to_vocab[value] = word\n",
    "\n",
    "usage_ratio = round(len(vocab_to_int) / len(word_counts),4)*100\n",
    "\n",
    "print(\"Total number of unique words:\", len(word_counts))\n",
    "print(\"Number of words we will use:\", len(vocab_to_int))\n",
    "print(\"Percent of words we will use: {}%\".format(usage_ratio))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63003\n"
     ]
    }
   ],
   "source": [
    "# Need to use 300 for embedding dimensions to match CN's vectors.\n",
    "embedding_dim = 300\n",
    "nb_words = len(vocab_to_int)\n",
    "\n",
    "# Create matrix with default values of zero\n",
    "word_embedding_matrix = np.zeros((nb_words, embedding_dim), dtype=np.float32)\n",
    "for word, i in vocab_to_int.items():\n",
    "    if word in embeddings_index:\n",
    "        word_embedding_matrix[i] = embeddings_index[word]\n",
    "    else:\n",
    "        # If word not in CN, create a random embedding for it\n",
    "        new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "        embeddings_index[word] = new_embedding\n",
    "        word_embedding_matrix[i] = new_embedding\n",
    "\n",
    "# Check if value matches len(vocab_to_int)\n",
    "print(len(word_embedding_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_to_ints(text, word_count, unk_count, eos=False):\n",
    "    '''Convert words in text to an integer.\n",
    "       If word is not in vocab_to_int, use UNK's integer.\n",
    "       Total the number of words and UNKs.\n",
    "       Add EOS token to the end of texts'''\n",
    "    ints = []\n",
    "    for sentence in text:\n",
    "        sentence_ints = []\n",
    "        for word in sentence.split():\n",
    "            word_count += 1\n",
    "            if word in vocab_to_int:\n",
    "                sentence_ints.append(vocab_to_int[word])\n",
    "            else:\n",
    "                sentence_ints.append(vocab_to_int[\"<UNK>\"])\n",
    "                unk_count += 1\n",
    "        if eos:\n",
    "            sentence_ints.append(vocab_to_int[\"<EOS>\"])\n",
    "        ints.append(sentence_ints)\n",
    "    return ints, word_count, unk_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of words in headlines: 23244340\n",
      "Total number of UNKs in headlines: 41997\n",
      "Percent of words that are UNK: 0.18%\n"
     ]
    }
   ],
   "source": [
    "# Apply convert_to_ints to clean_summaries and clean_texts\n",
    "word_count = 0\n",
    "unk_count = 0\n",
    "\n",
    "int_summaries, word_count, unk_count = convert_to_ints(clean_summaries, word_count, unk_count)\n",
    "int_texts, word_count, unk_count = convert_to_ints(clean_texts, word_count, unk_count, eos=True)\n",
    "\n",
    "unk_percent = round(unk_count/word_count,4)*100\n",
    "\n",
    "print(\"Total number of words in headlines:\", word_count)\n",
    "print(\"Total number of UNKs in headlines:\", unk_count)\n",
    "print(\"Percent of words that are UNK: {}%\".format(unk_percent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_lengths(text):\n",
    "    '''Create a data frame of the sentence lengths from a text'''\n",
    "    lengths = []\n",
    "    for sentence in text:\n",
    "        lengths.append(len(sentence))\n",
    "    return pd.DataFrame(lengths, columns=['counts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summaries:\n",
      "             counts\n",
      "count  83568.000000\n",
      "mean      41.437321\n",
      "std        9.833400\n",
      "min        8.000000\n",
      "25%       34.000000\n",
      "50%       42.000000\n",
      "75%       49.000000\n",
      "max      104.000000\n",
      "\n",
      "Texts:\n",
      "             counts\n",
      "count  83568.000000\n",
      "mean     237.711492\n",
      "std      128.070707\n",
      "min        1.000000\n",
      "25%      137.000000\n",
      "50%      218.000000\n",
      "75%      316.000000\n",
      "max      799.000000\n"
     ]
    }
   ],
   "source": [
    "lengths_summaries = create_lengths(int_summaries)\n",
    "lengths_texts = create_lengths(int_texts)\n",
    "\n",
    "print(\"Summaries:\")\n",
    "print(lengths_summaries.describe())\n",
    "print()\n",
    "print(\"Texts:\")\n",
    "print(lengths_texts.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "416.0\n",
      "484.0\n",
      "597.33\n"
     ]
    }
   ],
   "source": [
    "# Inspect the length of texts\n",
    "print(np.percentile(lengths_texts.counts, 90))\n",
    "print(np.percentile(lengths_texts.counts, 95))\n",
    "print(np.percentile(lengths_texts.counts, 99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54.0\n",
      "57.0\n",
      "62.0\n"
     ]
    }
   ],
   "source": [
    "# Inspect the length of summaries\n",
    "print(np.percentile(lengths_summaries.counts, 90))\n",
    "print(np.percentile(lengths_summaries.counts, 95))\n",
    "print(np.percentile(lengths_summaries.counts, 99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unk_counter(sentence):\n",
    "    '''Counts the number of time UNK appears in a sentence.'''\n",
    "    unk_count = 0\n",
    "    for word in sentence:\n",
    "        if word == vocab_to_int[\"<UNK>\"]:\n",
    "            unk_count += 1\n",
    "    return unk_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66694\n",
      "66694\n"
     ]
    }
   ],
   "source": [
    "# Sort the summaries and texts by the length of the texts, shortest to longest\n",
    "# Limit the length of summaries and texts based on the min and max ranges.\n",
    "# Remove reviews that include too many UNKs\n",
    "\n",
    "sorted_summaries = []\n",
    "sorted_texts = []\n",
    "max_text_length = 416\n",
    "max_summary_length = 62\n",
    "min_length = 2\n",
    "unk_text_limit = 1\n",
    "unk_summary_limit = 0\n",
    "\n",
    "for length in range(min(lengths_texts.counts), max_text_length): \n",
    "    for count, words in enumerate(int_summaries):\n",
    "        if (len(int_summaries[count]) >= min_length and\n",
    "            len(int_summaries[count]) <= max_summary_length and\n",
    "            len(int_texts[count]) >= min_length and\n",
    "            unk_counter(int_summaries[count]) <= unk_summary_limit and\n",
    "            unk_counter(int_texts[count]) <= unk_text_limit and\n",
    "            length == len(int_texts[count])\n",
    "           ):\n",
    "            sorted_summaries.append(int_summaries[count])\n",
    "            sorted_texts.append(int_texts[count])\n",
    "        \n",
    "# Compare lengths to ensure they match\n",
    "print(len(sorted_summaries))\n",
    "print(len(sorted_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    '''Create palceholders for inputs to the model'''\n",
    "    \n",
    "    input_data = tf.placeholder(tf.int32, [None, None], name='input')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    lr = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "    summary_length = tf.placeholder(tf.int32, (None,), name='summary_length')\n",
    "    max_summary_length = tf.reduce_max(summary_length, name='max_dec_len')\n",
    "    text_length = tf.placeholder(tf.int32, (None,), name='text_length')\n",
    "\n",
    "    return input_data, targets, lr, keep_prob, summary_length, max_summary_length, text_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_encoding_input(target_data, vocab_to_int, batch_size):\n",
    "    '''Remove the last word id from each batch and concat the <GO> to the begining of each batch'''\n",
    "    \n",
    "    ending = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    dec_input = tf.concat([tf.fill([batch_size, 1], vocab_to_int['<GO>']), ending], 1)\n",
    "\n",
    "    return dec_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def encoding_layer(rnn_size, sequence_length, num_layers, rnn_inputs, keep_prob):\n",
    "    '''Create the encoding layer'''\n",
    "    \n",
    "    for layer in range(num_layers):\n",
    "        with tf.variable_scope('encoder_{}'.format(layer)):\n",
    "            cell_fw = tf.contrib.rnn.LSTMCell(rnn_size,\n",
    "                                              initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))\n",
    "            cell_fw = tf.contrib.rnn.DropoutWrapper(cell_fw, \n",
    "                                                    input_keep_prob = keep_prob)\n",
    "\n",
    "            cell_bw = tf.contrib.rnn.LSTMCell(rnn_size,\n",
    "                                              initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))\n",
    "            cell_bw = tf.contrib.rnn.DropoutWrapper(cell_bw, \n",
    "                                                    input_keep_prob = keep_prob)\n",
    "\n",
    "            enc_output, enc_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, \n",
    "                                                                    cell_bw, \n",
    "                                                                    rnn_inputs,\n",
    "                                                                    sequence_length,\n",
    "                                                                    dtype=tf.float32)\n",
    "    # Join outputs since we are using a bidirectional RNN\n",
    "    enc_output = tf.concat(enc_output,2)\n",
    "    \n",
    "    return enc_output, enc_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def training_decoding_layer(dec_embed_input, summary_length, dec_cell, initial_state, output_layer, \n",
    "                            vocab_size, max_summary_length):\n",
    "    '''Create the training logits'''\n",
    "    \n",
    "    training_helper = tf.contrib.seq2seq.TrainingHelper(inputs=dec_embed_input,\n",
    "                                                        sequence_length=summary_length,\n",
    "                                                        time_major=False)\n",
    "\n",
    "    training_decoder = tf.contrib.seq2seq.BasicDecoder(dec_cell,\n",
    "                                                       training_helper,\n",
    "                                                       initial_state,\n",
    "                                                       output_layer) \n",
    "\n",
    "    training_logits, _ = tf.contrib.seq2seq.dynamic_decode(training_decoder,\n",
    "                                                           output_time_major=False,\n",
    "                                                           impute_finished=True,\n",
    "                                                           maximum_iterations=max_summary_length)\n",
    "    return training_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def inference_decoding_layer(embeddings, start_token, end_token, dec_cell, initial_state, output_layer,\n",
    "                             max_summary_length, batch_size):\n",
    "    '''Create the inference logits'''\n",
    "    \n",
    "    start_tokens = tf.tile(tf.constant([start_token], dtype=tf.int32), [batch_size], name='start_tokens')\n",
    "    \n",
    "    inference_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(embeddings,\n",
    "                                                                start_tokens,\n",
    "                                                                end_token)\n",
    "                \n",
    "    inference_decoder = tf.contrib.seq2seq.BasicDecoder(dec_cell,\n",
    "                                                        inference_helper,\n",
    "                                                        initial_state,\n",
    "                                                        output_layer)\n",
    "                \n",
    "    inference_logits, _ = tf.contrib.seq2seq.dynamic_decode(inference_decoder,\n",
    "                                                            output_time_major=False,\n",
    "                                                            impute_finished=True,\n",
    "                                                            maximum_iterations=max_summary_length)\n",
    "    \n",
    "    return inference_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decoding_layer(dec_embed_input, embeddings, enc_output, enc_state, vocab_size, text_length, summary_length, \n",
    "                   max_summary_length, rnn_size, vocab_to_int, keep_prob, batch_size, num_layers):\n",
    "    '''Create the decoding cell and attention for the training and inference decoding layers'''\n",
    "    \n",
    "    for layer in range(num_layers):\n",
    "        with tf.variable_scope('decoder_{}'.format(layer)):\n",
    "            lstm = tf.contrib.rnn.LSTMCell(rnn_size,\n",
    "                                           initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))\n",
    "            dec_cell = tf.contrib.rnn.DropoutWrapper(lstm, \n",
    "                                                     input_keep_prob = keep_prob)\n",
    "    \n",
    "    output_layer = Dense(vocab_size,\n",
    "                         kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))\n",
    "    \n",
    "    attn_mech = tf.contrib.seq2seq.BahdanauAttention(rnn_size,\n",
    "                                                  enc_output,\n",
    "                                                  text_length,\n",
    "                                                  normalize=False,\n",
    "                                                  name='BahdanauAttention')\n",
    "\n",
    "    dec_cell = tf.contrib.seq2seq.DynamicAttentionWrapper(dec_cell,\n",
    "                                                          attn_mech,\n",
    "                                                          rnn_size)\n",
    "            \n",
    "    initial_state = tf.contrib.seq2seq.DynamicAttentionWrapperState(enc_state[0],\n",
    "                                                                    _zero_state_tensors(rnn_size, \n",
    "                                                                                        batch_size, \n",
    "                                                                                        tf.float32)) \n",
    "    with tf.variable_scope(\"decode\"):\n",
    "        training_logits = training_decoding_layer(dec_embed_input, \n",
    "                                                  summary_length, \n",
    "                                                  dec_cell, \n",
    "                                                  initial_state,\n",
    "                                                  output_layer,\n",
    "                                                  vocab_size, \n",
    "                                                  max_summary_length)\n",
    "    with tf.variable_scope(\"decode\", reuse=True):\n",
    "        inference_logits = inference_decoding_layer(embeddings,  \n",
    "                                                    vocab_to_int['<GO>'], \n",
    "                                                    vocab_to_int['<EOS>'],\n",
    "                                                    dec_cell, \n",
    "                                                    initial_state, \n",
    "                                                    output_layer,\n",
    "                                                    max_summary_length,\n",
    "                                                    batch_size)\n",
    "\n",
    "    return training_logits, inference_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def seq2seq_model(input_data, target_data, keep_prob, text_length, summary_length, max_summary_length, \n",
    "                  vocab_size, rnn_size, num_layers, vocab_to_int, batch_size):\n",
    "    '''Use the previous functions to create the training and inference logits'''\n",
    "    \n",
    "    # Use Numberbatch's embeddings and the newly created ones as our embeddings\n",
    "    embeddings = word_embedding_matrix\n",
    "    \n",
    "    enc_embed_input = tf.nn.embedding_lookup(embeddings, input_data)\n",
    "    enc_output, enc_state = encoding_layer(rnn_size, text_length, num_layers, enc_embed_input, keep_prob)\n",
    "    \n",
    "    dec_input = process_encoding_input(target_data, vocab_to_int, batch_size)\n",
    "    dec_embed_input = tf.nn.embedding_lookup(embeddings, dec_input)\n",
    "    \n",
    "    training_logits, inference_logits  = decoding_layer(dec_embed_input, \n",
    "                                                        embeddings,\n",
    "                                                        enc_output,\n",
    "                                                        enc_state, \n",
    "                                                        vocab_size, \n",
    "                                                        text_length, \n",
    "                                                        summary_length, \n",
    "                                                        max_summary_length,\n",
    "                                                        rnn_size, \n",
    "                                                        vocab_to_int, \n",
    "                                                        keep_prob, \n",
    "                                                        batch_size,\n",
    "                                                        num_layers)\n",
    "    \n",
    "    return training_logits, inference_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch):\n",
    "    \"\"\"Pad sentences with <PAD> so that each sentence of a batch has the same length\"\"\"\n",
    "    max_sentence = max([len(sentence) for sentence in sentence_batch])\n",
    "    return [sentence + [vocab_to_int['<PAD>']] * (max_sentence - len(sentence)) for sentence in sentence_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batches(summaries, texts, batch_size):\n",
    "    \"\"\"Batch summaries, texts, and the lengths of their sentences together\"\"\"\n",
    "    for batch_i in range(0, len(texts)//batch_size):\n",
    "        start_i = batch_i * batch_size\n",
    "        summaries_batch = summaries[start_i:start_i + batch_size]\n",
    "        texts_batch = texts[start_i:start_i + batch_size]\n",
    "        pad_summaries_batch = np.array(pad_sentence_batch(summaries_batch))\n",
    "        pad_texts_batch = np.array(pad_sentence_batch(texts_batch))\n",
    "        \n",
    "        # Need the lengths for the _lengths parameters\n",
    "        pad_summaries_lengths = []\n",
    "        for summary in pad_summaries_batch:\n",
    "            pad_summaries_lengths.append(len(summary))\n",
    "        \n",
    "        pad_texts_lengths = []\n",
    "        for text in pad_texts_batch:\n",
    "            pad_texts_lengths.append(len(text))\n",
    "        \n",
    "        yield pad_summaries_batch, pad_texts_batch, pad_summaries_lengths, pad_texts_lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set the Hyperparameters\n",
    "epochs = 100\n",
    "batch_size = 64\n",
    "rnn_size = 256\n",
    "num_layers = 2\n",
    "learning_rate = 0.005\n",
    "keep_probability = 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Graph is built.\n"
     ]
    }
   ],
   "source": [
    "# Build the graph\n",
    "train_graph = tf.Graph()\n",
    "# Set the graph to default to ensure that it is ready for training\n",
    "with train_graph.as_default():\n",
    "    \n",
    "    # Load the model inputs    \n",
    "    input_data, targets, lr, keep_prob, summary_length, max_summary_length, text_length = model_inputs()\n",
    "\n",
    "    # Create the training and inference logits\n",
    "    training_logits, inference_logits = seq2seq_model(tf.reverse(input_data, [-1]),\n",
    "                                                      targets, \n",
    "                                                      keep_prob,   \n",
    "                                                      text_length,\n",
    "                                                      summary_length,\n",
    "                                                      max_summary_length,\n",
    "                                                      len(vocab_to_int)+1,\n",
    "                                                      rnn_size, \n",
    "                                                      num_layers, \n",
    "                                                      vocab_to_int,\n",
    "                                                      batch_size)\n",
    "    \n",
    "    # Create tensors for the training logits and inference logits\n",
    "    training_logits = tf.identity(training_logits.rnn_output, 'logits')\n",
    "    inference_logits = tf.identity(inference_logits.sample_id, name='predictions')\n",
    "    \n",
    "    # Create the weights for sequence_loss\n",
    "    masks = tf.sequence_mask(summary_length, max_summary_length, dtype=tf.float32, name='masks')\n",
    "\n",
    "    with tf.name_scope(\"optimization\"):\n",
    "        # Loss function\n",
    "        cost = tf.contrib.seq2seq.sequence_loss(\n",
    "            training_logits,\n",
    "            targets,\n",
    "            masks)\n",
    "\n",
    "        # Optimizer\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "        # Gradient Clipping\n",
    "        gradients = optimizer.compute_gradients(cost)\n",
    "        capped_gradients = [(tf.clip_by_value(grad, -5., 5.), var) for grad, var in gradients if grad is not None]\n",
    "        train_op = optimizer.apply_gradients(capped_gradients)\n",
    "print(\"Graph is built.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shortest text length: 5\n",
      "The longest text length: 415\n"
     ]
    }
   ],
   "source": [
    "# Subset the data for training\n",
    "# start = 8000\n",
    "# end = start + 2000\n",
    "sorted_summaries_short = sorted_summaries\n",
    "sorted_texts_short = sorted_texts\n",
    "print(\"The shortest text length:\", len(sorted_texts_short[0]))\n",
    "print(\"The longest text length:\",len(sorted_texts_short[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "cannot add op with name encoder_1/bidirectional_rnn/fw/lstm_cell/weights/Adam as that name is already used",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m-----------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-55-034a6bdf6681>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0;31m# If we want to continue training a previous session\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m     \u001b[0mloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_meta_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"./\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcheckpoint\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.meta'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m     \u001b[0mloader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mimport_meta_graph\u001b[0;34m(meta_graph_or_file, clear_devices, import_scope, **kwargs)\u001b[0m\n\u001b[1;32m   1593\u001b[0m                                       \u001b[0mclear_devices\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclear_devices\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1594\u001b[0m                                       \u001b[0mimport_scope\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mimport_scope\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1595\u001b[0;31m                                       **kwargs)\n\u001b[0m\u001b[1;32m   1596\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mmeta_graph_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHasField\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"saver_def\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1597\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mSaver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msaver_def\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmeta_graph_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaver_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mimport_scope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/meta_graph.py\u001b[0m in \u001b[0;36mimport_scoped_meta_graph\u001b[0;34m(meta_graph_or_file, clear_devices, graph, import_scope, input_map, unbound_inputs_col_name)\u001b[0m\n\u001b[1;32m    497\u001b[0m     importer.import_graph_def(\n\u001b[1;32m    498\u001b[0m         \u001b[0minput_graph_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimport_scope\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_map\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_map\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 499\u001b[0;31m         producer_op_list=producer_op_list)\n\u001b[0m\u001b[1;32m    500\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    501\u001b[0m     \u001b[0;31m# Restores all the other collections.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/importer.py\u001b[0m in \u001b[0;36mimport_graph_def\u001b[0;34m(graph_def, input_map, return_elements, name, op_dict, producer_op_list)\u001b[0m\n\u001b[1;32m    306\u001b[0m           \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_types\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m           \u001b[0mcompute_shapes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompute_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 308\u001b[0;31m           op_def=op_def)\n\u001b[0m\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m     \u001b[0;31m# 2. Add inputs to the operations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mcreate_op\u001b[0;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_shapes, compute_device)\u001b[0m\n\u001b[1;32m   2337\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcompute_shapes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2338\u001b[0m       \u001b[0mset_shapes_for_outputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2339\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_add_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2340\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_record_op_seen_by_control_dependencies\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_add_op\u001b[0;34m(self, op)\u001b[0m\n\u001b[1;32m   2031\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nodes_by_name\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2032\u001b[0m         raise ValueError(\"cannot add op with name %s as that name \"\n\u001b[0;32m-> 2033\u001b[0;31m                          \"is already used\" % op.name)\n\u001b[0m\u001b[1;32m   2034\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nodes_by_id\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2035\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nodes_by_name\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: cannot add op with name encoder_1/bidirectional_rnn/fw/lstm_cell/weights/Adam as that name is already used"
     ]
    }
   ],
   "source": [
    "# Train the Model\n",
    "learning_rate_decay = 0.95\n",
    "min_learning_rate = 0.0005\n",
    "display_step = 20 # Check training loss after every 20 batches\n",
    "stop_early = 0 \n",
    "stop = 3 # If the update loss does not decrease in 3 consecutive update checks, stop training\n",
    "per_epoch = 3 # Make 3 update checks per epoch\n",
    "update_check = (len(sorted_texts_short)//batch_size//per_epoch)-1\n",
    "\n",
    "update_loss = 0 \n",
    "batch_loss = 0\n",
    "summary_update_loss = [] # Record the update losses for saving improvements in the model\n",
    "\n",
    "checkpoint = \"best_model.ckpt\" \n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    # If we want to continue training a previous session\n",
    "    loader = tf.train.import_meta_graph(\"./\" + checkpoint + '.meta')\n",
    "    loader.restore(sess, checkpoint)\n",
    "    \n",
    "    for epoch_i in range(1, epochs+1):\n",
    "        update_loss = 0\n",
    "        batch_loss = 0\n",
    "        for batch_i, (summaries_batch, texts_batch, summaries_lengths, texts_lengths) in enumerate(\n",
    "                get_batches(sorted_summaries_short, sorted_texts_short, batch_size)):\n",
    "            start_time = time.time()\n",
    "            _, loss = sess.run(\n",
    "                [train_op, cost],\n",
    "                {input_data: texts_batch,\n",
    "                 targets: summaries_batch,\n",
    "                 lr: learning_rate,\n",
    "                 summary_length: summaries_lengths,\n",
    "                 text_length: texts_lengths,\n",
    "                 keep_prob: keep_probability})\n",
    "\n",
    "            batch_loss += loss\n",
    "            update_loss += loss\n",
    "            end_time = time.time()\n",
    "            batch_time = end_time - start_time\n",
    "\n",
    "            if batch_i % display_step == 0 and batch_i > 0:\n",
    "                print('Epoch {:>3}/{} Batch {:>4}/{} - Loss: {:>6.3f}, Seconds: {:>4.2f}'\n",
    "                      .format(epoch_i,\n",
    "                              epochs, \n",
    "                              batch_i, \n",
    "                              len(sorted_texts_short) // batch_size, \n",
    "                              batch_loss / display_step, \n",
    "                              batch_time*display_step))\n",
    "                batch_loss = 0\n",
    "\n",
    "            if batch_i % update_check == 0 and batch_i > 0:\n",
    "                print(\"Average loss for this update:\", round(update_loss/update_check,3))\n",
    "                summary_update_loss.append(update_loss)\n",
    "                \n",
    "                # If the update loss is at a new minimum, save the model\n",
    "                if update_loss <= min(summary_update_loss):\n",
    "                    print('New Record!') \n",
    "                    stop_early = 0\n",
    "                    saver = tf.train.Saver() \n",
    "                    saver.save(sess, checkpoint)\n",
    "\n",
    "                else:\n",
    "                    print(\"No Improvement.\")\n",
    "                    stop_early += 1\n",
    "                    if stop_early == stop:\n",
    "                        break\n",
    "                update_loss = 0\n",
    "            \n",
    "                    \n",
    "        # Reduce learning rate, but not below its minimum value\n",
    "        learning_rate *= learning_rate_decay\n",
    "        if learning_rate < min_learning_rate:\n",
    "            learning_rate = min_learning_rate\n",
    "        \n",
    "        if stop_early == stop:\n",
    "            print(\"Stopping Training.\")\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def text_to_seq(text):\n",
    "    '''Prepare the text for the model'''\n",
    "    \n",
    "    text = clean_text(text)\n",
    "    return [vocab_to_int.get(word, vocab_to_int['<UNK>']) for word in text.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1093\n",
      "1093\n"
     ]
    }
   ],
   "source": [
    "path = \"val_test/cnn/test\" #文件夹目录  \n",
    "files= os.listdir(path) #得到文件夹下的所有文件名称 \n",
    "all_test_article = [] \n",
    "all_test_target = []\n",
    "for file in files: #遍历文件夹  \n",
    "    if not os.path.isdir(file): #判断是否是文件夹，不是文件夹才打开  \n",
    "        f = open(path+\"/\"+file); #打开文件  \n",
    "        iter_f = iter(f); #创建迭代器  \n",
    "        tab = ' '\n",
    "        lines = []\n",
    "        target = []\n",
    "        n_count = 0\n",
    "        for line in iter_f: #遍历文件，一行行遍历，读取文本  \n",
    "            if line.strip()!='' and line.find('\\t0')==-1 and line.find('\\t\\t\\t')!=-1: # 去掉标记为0的sentence\n",
    "                lines.append(line.strip()[:-4])\n",
    "            elif line.find('\\t\\t\\t')==-1:\n",
    "                if line == '\\n':\n",
    "                    n_count+=1\n",
    "                else:\n",
    "                    if n_count >= 3:\n",
    "                        break;\n",
    "                    target.append(line.strip())\n",
    "        lines = tab.join(lines)\n",
    "        target = target[1:]\n",
    "        target = tab.join(target)\n",
    "        all_test_article.append(lines) #每个文件的文本存到list中  \n",
    "        all_test_target.append(target)\n",
    "        \n",
    "print(len(all_test_article)) \n",
    "print(len(all_test_target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = pd.DataFrame({'Summary':all_test_target,'Text':all_test_article})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create your own review or use one from the dataset\n",
    "#input_sentence = \"I have never eaten an apple before, but this red one was nice. \\\n",
    "                  #I think that I will try a green apple next time.\"\n",
    "#text = text_to_seq(input_sentence)\n",
    "def test_model(input_sentence):\n",
    "#     random = np.random.randint(0,len(clean_texts))\n",
    "#     input_sentence = clean_texts[random]\n",
    "#     text = text_to_seq(clean_texts[random])\n",
    "    text = text_to_seq(input_sentence)\n",
    "\n",
    "    checkpoint = \"./best_model.ckpt\"\n",
    "\n",
    "    loaded_graph = tf.Graph()\n",
    "    with tf.Session(graph=loaded_graph) as sess:\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(checkpoint + '.meta')\n",
    "        loader.restore(sess, checkpoint)\n",
    "\n",
    "        input_data = loaded_graph.get_tensor_by_name('input:0')\n",
    "        logits = loaded_graph.get_tensor_by_name('predictions:0')\n",
    "        text_length = loaded_graph.get_tensor_by_name('text_length:0')\n",
    "        summary_length = loaded_graph.get_tensor_by_name('summary_length:0')\n",
    "        keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')\n",
    "    \n",
    "        #Multiply by batch_size to match the model's input parameters\n",
    "        answer_logits = sess.run(logits, {input_data: [text]*batch_size, \n",
    "                                          summary_length: [np.random.randint(25,35)], \n",
    "                                          text_length: [len(text)]*batch_size,\n",
    "                                          keep_prob: 1.0})[0] \n",
    "\n",
    "    # Remove the padding from the tweet\n",
    "    pad = vocab_to_int[\"<PAD>\"] \n",
    "    \n",
    "    re = \" \".join([int_to_vocab[i] for i in answer_logits if i != pad])\n",
    "\n",
    "#     print('Original Text:', input_sentence)\n",
    "\n",
    "#     print('\\nText')\n",
    "#     print('  Word Ids:    {}'.format([i for i in text]))\n",
    "#     print('  Input Words: {}'.format(\" \".join([int_to_vocab[i] for i in text])))\n",
    "\n",
    "#     print('\\nSummary')\n",
    "#     print('  Word Ids:       {}'.format([i for i in answer_logits if i != pad]))\n",
    "    print('  Response Words: {}'.format(\" \".join([int_to_vocab[i] for i in answer_logits if i != pad])))\n",
    "    return re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./best_model.ckpt\n",
      "  Response Words: entity1 is a great life in the entity4 entity1 was born in entity12 in entity12 entity13 was born in entity12 in entity12 entity13 was born in entity12 in\n",
      "INFO:tensorflow:Restoring parameters from ./best_model.ckpt\n",
      "  Response Words: entity7 is a great series of the entity7 s most famous entity4 film the entity7 is a great role of the entity7 s entity17 and entity17\n",
      "INFO:tensorflow:Restoring parameters from ./best_model.ckpt\n",
      "  Response Words: the ad is a ad ad in the entity3 campaign ads the ad is a key ad in the same election the ad is the first of the most women who are among\n",
      "INFO:tensorflow:Restoring parameters from ./best_model.ckpt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-c6593a392b24>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0minput_sentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# #     print(input_sentence)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_sentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;31m#     print(result)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Summary_model'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-40-c4fce1e13a1d>\u001b[0m in \u001b[0;36mtest_model\u001b[0;34m(input_sentence)\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0;31m# Load saved model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_meta_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.meta'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0mloader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0minput_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloaded_graph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_tensor_by_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'input:0'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, sess, save_path)\u001b[0m\n\u001b[1;32m   1455\u001b[0m     \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Restoring parameters from %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1456\u001b[0m     sess.run(self.saver_def.restore_op_name,\n\u001b[0;32m-> 1457\u001b[0;31m              {self.saver_def.filename_tensor_name: save_path})\n\u001b[0m\u001b[1;32m   1458\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1459\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    776\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    777\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 778\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    779\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    980\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    981\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 982\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    983\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    984\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1030\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1031\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1032\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1033\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1034\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1037\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1040\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1015\u001b[0m                 run_metadata):\n\u001b[1;32m   1016\u001b[0m       \u001b[0;31m# Ensure any changes to the graph are reflected in the runtime.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1017\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1018\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m         return tf_session.TF_Run(session, options,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_extend_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1064\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1065\u001b[0m           tf_session.TF_ExtendGraph(\n\u001b[0;32m-> 1066\u001b[0;31m               self._session, graph_def.SerializeToString(), status)\n\u001b[0m\u001b[1;32m   1067\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_opened\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1068\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/python_message.py\u001b[0m in \u001b[0;36mSerializeToString\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m   1040\u001b[0m           'Message %s is missing required fields: %s' % (\n\u001b[1;32m   1041\u001b[0m           self.DESCRIPTOR.full_name, ','.join(self.FindInitializationErrors())))\n\u001b[0;32m-> 1042\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializePartialToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1043\u001b[0m   \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializeToString\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSerializeToString\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1044\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/python_message.py\u001b[0m in \u001b[0;36mSerializePartialToString\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mSerializePartialToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1050\u001b[0m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1051\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_InternalSerialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m   \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializePartialToString\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSerializePartialToString\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/python_message.py\u001b[0m in \u001b[0;36mInternalSerialize\u001b[0;34m(self, write_bytes, deterministic)\u001b[0m\n\u001b[1;32m   1069\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1070\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0mfield_descriptor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfield_value\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mListFields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1071\u001b[0;31m         \u001b[0mfield_descriptor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_encoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrite_bytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfield_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1072\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0mtag_bytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_bytes\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unknown_fields\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1073\u001b[0m         \u001b[0mwrite_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag_bytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/encoder.py\u001b[0m in \u001b[0;36mEncodeRepeatedField\u001b[0;34m(write, value, deterministic)\u001b[0m\n\u001b[1;32m    757\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0melement\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m         \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 759\u001b[0;31m         \u001b[0mlocal_EncodeVarint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0melement\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mByteSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    760\u001b[0m         \u001b[0melement\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_InternalSerialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    761\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mEncodeRepeatedField\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/python_message.py\u001b[0m in \u001b[0;36mByteSize\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1018\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0mfield_descriptor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfield_value\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mListFields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1020\u001b[0;31m         \u001b[0msize\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mfield_descriptor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfield_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1021\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0mtag_bytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_bytes\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unknown_fields\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1022\u001b[0m         \u001b[0msize\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag_bytes\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue_bytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/encoder.py\u001b[0m in \u001b[0;36mFieldSize\u001b[0;34m(map_value)\u001b[0m\n\u001b[1;32m    359\u001b[0m       \u001b[0;31m# update the status.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m       \u001b[0mentry_msg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmessage_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_concrete_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 361\u001b[0;31m       \u001b[0mtotal\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mmessage_sizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mentry_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    362\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mis_message_map\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    363\u001b[0m         \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mByteSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/encoder.py\u001b[0m in \u001b[0;36mFieldSize\u001b[0;34m(value)\u001b[0m\n\u001b[1;32m    306\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mFieldSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 308\u001b[0;31m       \u001b[0ml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mByteSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    309\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mtag_size\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlocal_VarintSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mFieldSize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/python_message.py\u001b[0m in \u001b[0;36mByteSize\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1015\u001b[0m       \u001b[0;31m# Fields of map entry should always be serialized.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       \u001b[0msize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdescriptor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfields_by_name\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'key'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1017\u001b[0;31m       \u001b[0msize\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdescriptor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfields_by_name\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'value'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1018\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0mfield_descriptor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfield_value\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mListFields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/encoder.py\u001b[0m in \u001b[0;36mFieldSize\u001b[0;34m(value)\u001b[0m\n\u001b[1;32m    306\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mFieldSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 308\u001b[0;31m       \u001b[0ml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mByteSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    309\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mtag_size\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlocal_VarintSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mFieldSize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/google/protobuf/internal/python_message.py\u001b[0m in \u001b[0;36mByteSize\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1019\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0mfield_descriptor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfield_value\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mListFields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m         \u001b[0msize\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mfield_descriptor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfield_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mfor\u001b[0m \u001b[0mtag_bytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_bytes\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unknown_fields\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m         \u001b[0msize\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag_bytes\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue_bytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for index,row in test_data.iterrows():\n",
    "#     print(row['Text'])\n",
    "    input_sentence = row['Text']\n",
    "# #     print(input_sentence)\n",
    "    result = test_model(input_sentence)\n",
    "#     print(result)\n",
    "    row['Summary_model'] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}