20170720-building-social-network-graphs-CSV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exports nodes and edges from tweets (Retweets, Mentions, or Replies) [CSV]\n",
    "Exports nodes and edges from tweets (either from retweets or mentions) in json format that can be exported from SFM, and saves it in a file format compatible with various social network graph tools such as Gephi, Cytoscape, Kumu, etc. These are for directed graphs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import json\n",
    "import re\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import pandas as pd  \n",
    "\n",
    "tweetfile = '/home/soominpark/sfmproject/Work/Network Graphs/food_security.csv'\n",
    "\n",
    "tweets = pd.read_csv(tweetfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Export edges from Retweets, Mentions, or Replies\n",
    "* Run one of three blocks of codes below for your purpose. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/soominpark/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "# 1. Export edges from Retweets\n",
    "\n",
    "retweets = tweets[tweets['is_retweet'] == 'Yes']\n",
    "retweets['original_twitter'] = retweets['text'].str.extract('RT @([a-zA-Z0-9]\\w{0,}):', expand=True)\n",
    "\n",
    "edges = retweets[['screen_name', 'original_twitter','created_at']]\n",
    "edges.columns = ['Source', 'Target', 'Strength']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 2. Export edges from Mentions\n",
    "\n",
    "mentions = tweets[tweets['mentions'].notnull()]\n",
    "\n",
    "edges = pd.DataFrame(columns=('Source','Target','Strength'))\n",
    "\n",
    "for index, row in mentions.iterrows():\n",
    "    mention_list = row['mentions'].split(\", \")\n",
    "    for mention in mention_list:\n",
    "        edges = edges.append(pd.DataFrame([[row['screen_name'],\n",
    "                                        mention,\n",
    "                                        row['created_at']]]\n",
    "                                        , columns=('Source','Target','Strength')), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 3. Export edges from Replies\n",
    "\n",
    "replies = tweets[tweets['in_reply_to_screen_name'].notnull()]\n",
    "\n",
    "edges = replies[['screen_name', 'in_reply_to_screen_name','created_at']]\n",
    "edges.columns = ['Source', 'Target', 'Strength']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Leave only the tweets whose strength level >= user specified level (directed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "strengthLevel = 3  # Network connection strength level: the number of times in total each of the tweeters responded to or mentioned the other.\n",
    "                   # If you have 1 as the level, then all tweeters who mentioned or replied to another at least once will be displayed. But if you have 5, only those who have mentioned or responded to a particular tweeter at least 5 times will be displayed, which means that only the strongest bonds are shown.\n",
    "\n",
    "edges2 = edges.groupby(['Source','Target'])['Strength'].count()\n",
    "edges2 = edges2.reset_index()\n",
    "edges2 = edges2[edges2['Strength'] >= strengthLevel]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Export nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Export nodes from the edges and add node attributes for both Sources and Targets.\n",
    "\n",
    "users = tweets[['screen_name','followers_count','friends_count']]\n",
    "users = users.sort_values(['screen_name','followers_count'], ascending=[True, False])\n",
    "users = users.drop_duplicates(['screen_name'], keep='first') \n",
    "\n",
    "ids = edges2['Source'].append(edges2['Target']).to_frame()\n",
    "ids['Label'] = ids\n",
    "ids.columns = ['screen_name', 'Label']\n",
    "ids = ids.drop_duplicates(['screen_name'], keep='first') \n",
    "nodes = pd.merge(ids, users, on='screen_name', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(463, 4)\n",
      "(349, 3)\n"
     ]
    }
   ],
   "source": [
    "print(nodes.shape)\n",
    "print(edges2.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Export nodes and edges to csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# change column names for Kumu import (Run this when using Kumu)\n",
    "edges2.columns = ['From','To','Strength']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>screen_name</th>\n",
       "      <th>Label</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>104boild</td>\n",
       "      <td>104boild</td>\n",
       "      <td>3665.0</td>\n",
       "      <td>3641.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AapRajasthan_</td>\n",
       "      <td>AapRajasthan_</td>\n",
       "      <td>30.0</td>\n",
       "      <td>391.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ActuallyD0NG</td>\n",
       "      <td>ActuallyD0NG</td>\n",
       "      <td>415.0</td>\n",
       "      <td>457.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     screen_name          Label  followers_count  friends_count\n",
       "0       104boild       104boild           3665.0         3641.0\n",
       "1  AapRajasthan_  AapRajasthan_             30.0          391.0\n",
       "2   ActuallyD0NG   ActuallyD0NG            415.0          457.0"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print nodes to check\n",
    "nodes.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source</th>\n",
       "      <th>Target</th>\n",
       "      <th>Strength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>104boild</td>\n",
       "      <td>104boild</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>AapRajasthan_</td>\n",
       "      <td>neelesh403</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>606</th>\n",
       "      <td>AapRajasthan_</td>\n",
       "      <td>sardesairajdeep</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Source           Target  Strength\n",
       "45        104boild         104boild         5\n",
       "604  AapRajasthan_       neelesh403         3\n",
       "606  AapRajasthan_  sardesairajdeep         3"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print edges to check\n",
    "edges2.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Export nodes and edges to csv files\n",
    "nodes.to_csv('nodes.csv', encoding='utf-8', index=False)\n",
    "edges2.to_csv('edges.csv', encoding='utf-8', index=False)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}