collect_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tweepy\n",
    "import json\n",
    "import os.path"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# Loading json file with required credentials\n",
    "with open(\"credentials.json\") as f:\n",
    "    credentials = json.load(f)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "# Authentication\n",
    "client = tweepy.Client(bearer_token=credentials['BEARER_TOKEN'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# Creating empy dataframes for tweets and users\n",
    "tweets_df = pd.DataFrame()\n",
    "users_df = pd.DataFrame()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "# List of hashtags\n",
    "queries = ['#QatarWorldCup2022', '#Qatar2022', '#FIFAWorldCup']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "# Getting set of collected tweets' ids\n",
    "ids = pd.read_csv(f'./data/tweets.csv', usecols=['id'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "# Search parameters\n",
    "params = {\n",
    "    'max_results': 100,\n",
    "    'limit': 200,\n",
    "    'tweet_fields': ['id', 'text', 'public_metrics', 'author_id', 'created_at', 'lang'],\n",
    "    'user_fields': ['username', 'created_at', 'public_metrics', 'protected', 'verified'],\n",
    "    'expansions': ['author_id', 'entities.mentions.username'],  # expansions needed to collect tweets' author details\n",
    "    'since_id': ids.max()   # id of a last collected tweet\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "import time"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Tweet download stopped due to TooManyRequest exception. Wait 15 minutes...\n"
     ]
    }
   ],
   "source": [
    "# Collecting tweets for each given hashtag\n",
    "for query in queries:\n",
    "    try:\n",
    "        # Creating empty lists for tweets and users\n",
    "        tweet_list = list()\n",
    "        users_list = list()\n",
    "\n",
    "        # Getting tweets for a given hashtag without retweets\n",
    "        for chunk in tweepy.Paginator(client.search_recent_tweets, query=f'{query} -is:retweet', **params):\n",
    "\n",
    "            # Obtaining tweet's details including mentioned users' ids\n",
    "            for tweets in chunk.data:\n",
    "                tweet = [query,\n",
    "                         tweets.id,\n",
    "                         tweets.author_id,\n",
    "                         tweets.text,\n",
    "                         tweets.public_metrics['like_count'],\n",
    "                         tweets.public_metrics['reply_count'],\n",
    "                         tweets.public_metrics['retweet_count'],\n",
    "                         tweets.public_metrics['quote_count'],\n",
    "                         tweets.created_at,\n",
    "                         tweets.lang]\n",
    "\n",
    "                # Appending list of mentioned users' ids\n",
    "                mentions = None\n",
    "                if tweets.entities is not None:\n",
    "                    mentions = [mention['id'] for mention in tweets.entities['mentions']]\n",
    "                tweet.append(mentions)\n",
    "\n",
    "                tweet_list.append(tweet)\n",
    "\n",
    "            # Obtaining user's details\n",
    "            for user in chunk.includes['users']:\n",
    "                users_list.append([user.id,\n",
    "                                   user.username,\n",
    "                                   user.created_at,\n",
    "                                   user.public_metrics['followers_count'],\n",
    "                                   user.public_metrics['following_count'],\n",
    "                                   user.public_metrics['tweet_count'],\n",
    "                                   user.protected,\n",
    "                                   user.verified])\n",
    "\n",
    "        # Creating temporary dataframe out of collected tweets\n",
    "        temp_tweets = pd.DataFrame(tweet_list,\n",
    "                                   columns=['hashtag', 'id', 'author_id', 'text', 'like_count', 'reply_count',\n",
    "                                            'retweet_count',\n",
    "                                            'quote_count',\n",
    "                                            'created_at', 'lang', 'mentions'])\n",
    "\n",
    "        # Creating temporary dataframe out of collected users\n",
    "        temp_users = pd.DataFrame(users_list,\n",
    "                                  columns=['id', 'username', 'created_at', 'followers_count', 'following_count',\n",
    "                                           'tweet_count', 'protected', 'verified'])\n",
    "\n",
    "        # Concatenating temporary dataframes with final ones\n",
    "        tweets_df = pd.concat([tweets_df, temp_tweets])\n",
    "        users_df = pd.concat([users_df, temp_users])\n",
    "\n",
    "    except tweepy.TooManyRequests:\n",
    "        print('WARNING: Tweet download stopped due to TooManyRequest exception. Wait 15 minutes...')\n",
    "        time.sleep(15 * 60)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "                  hashtag                   id            author_id  \\\n0      #QatarWorldCup2022  1605584655431892995  1447603056359714817   \n1      #QatarWorldCup2022  1605584631612329985  1485652252002066437   \n2      #QatarWorldCup2022  1605584449877549058  1447604020462370822   \n3      #QatarWorldCup2022  1605584342943776768  1055130038092345344   \n4      #QatarWorldCup2022  1605583974289375232            191106942   \n...                   ...                  ...                  ...   \n19955          #Qatar2022  1605037030064685057           1530154136   \n19956          #Qatar2022  1605037015233310720  1363653333479788549   \n19957          #Qatar2022  1605036970660462602  1522221998151061509   \n19958          #Qatar2022  1605036968588484609  1504760689650552863   \n19959          #Qatar2022  1605036951786074113           4896706619   \n\n                                                    text  like_count  \\\n0      #Deportes |  El mundial de Qatar 2022 se carac...           1   \n1      Hadji: « La FRMF a fait le bon choix en parian...           0   \n2      #Deportes |  El mundial de Qatar 2022 se carac...           0   \n3      #Deportes |  El mundial de Qatar 2022 se carac...           0   \n4      📢 Popularité du couple exécutif, #QatarWorldCu...           0   \n...                                                  ...         ...   \n19955  代表チームの強さと人口は無関係である\\n#FIFAWorldCup #Qatar2022 #...           7   \n19956  Argentine dairy industry #DitchDairy @dairy_tr...           0   \n19957  This time @TeamMessi Argentina ⚽❤✌ \\n\\n 📕 #Dex...           0   \n19958  The real GOAT 🐐 congrats LEO @TeamMessi😭🇦🇷 \\n\\...           0   \n19959  #MessiCampeonMundial \\n#Qatar2022 \\n#Scaloneta...           0   \n\n       reply_count  retweet_count  quote_count                created_at lang  \\\n0                0              0            0 2022-12-21 15:23:06+00:00   es   \n1                0              0            0 2022-12-21 15:23:01+00:00   fr   \n2                0              0            0 2022-12-21 15:22:17+00:00   es   \n3                0              0            0 2022-12-21 15:21:52+00:00   es   \n4                0              0            0 2022-12-21 15:20:24+00:00   fr   \n...            ...            ...          ...                       ...  ...   \n19955            0              1            1 2022-12-20 03:07:02+00:00   ja   \n19956            0              0            0 2022-12-20 03:06:59+00:00   en   \n19957            0              0            0 2022-12-20 03:06:48+00:00   en   \n19958            0              0            0 2022-12-20 03:06:48+00:00   en   \n19959            0              0            0 2022-12-20 03:06:44+00:00  und   \n\n                                         mentions  \n0                                            None  \n1                                            None  \n2                                            None  \n3                                            None  \n4                           [296824934, 34294667]  \n...                                           ...  \n19955                                        None  \n19956  [1370074650038910983, 1246189682150367233]  \n19957                                [1058376110]  \n19958                                [1058376110]  \n19959                                        None  \n\n[39889 rows x 11 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>hashtag</th>\n      <th>id</th>\n      <th>author_id</th>\n      <th>text</th>\n      <th>like_count</th>\n      <th>reply_count</th>\n      <th>retweet_count</th>\n      <th>quote_count</th>\n      <th>created_at</th>\n      <th>lang</th>\n      <th>mentions</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>#QatarWorldCup2022</td>\n      <td>1605584655431892995</td>\n      <td>1447603056359714817</td>\n      <td>#Deportes |  El mundial de Qatar 2022 se carac...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-21 15:23:06+00:00</td>\n      <td>es</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>#QatarWorldCup2022</td>\n      <td>1605584631612329985</td>\n      <td>1485652252002066437</td>\n      <td>Hadji: « La FRMF a fait le bon choix en parian...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-21 15:23:01+00:00</td>\n      <td>fr</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>#QatarWorldCup2022</td>\n      <td>1605584449877549058</td>\n      <td>1447604020462370822</td>\n      <td>#Deportes |  El mundial de Qatar 2022 se carac...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-21 15:22:17+00:00</td>\n      <td>es</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>#QatarWorldCup2022</td>\n      <td>1605584342943776768</td>\n      <td>1055130038092345344</td>\n      <td>#Deportes |  El mundial de Qatar 2022 se carac...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-21 15:21:52+00:00</td>\n      <td>es</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>#QatarWorldCup2022</td>\n      <td>1605583974289375232</td>\n      <td>191106942</td>\n      <td>📢 Popularité du couple exécutif, #QatarWorldCu...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-21 15:20:24+00:00</td>\n      <td>fr</td>\n      <td>[296824934, 34294667]</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>19955</th>\n      <td>#Qatar2022</td>\n      <td>1605037030064685057</td>\n      <td>1530154136</td>\n      <td>代表チームの強さと人口は無関係である\\n#FIFAWorldCup #Qatar2022 #...</td>\n      <td>7</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2022-12-20 03:07:02+00:00</td>\n      <td>ja</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>19956</th>\n      <td>#Qatar2022</td>\n      <td>1605037015233310720</td>\n      <td>1363653333479788549</td>\n      <td>Argentine dairy industry #DitchDairy @dairy_tr...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-20 03:06:59+00:00</td>\n      <td>en</td>\n      <td>[1370074650038910983, 1246189682150367233]</td>\n    </tr>\n    <tr>\n      <th>19957</th>\n      <td>#Qatar2022</td>\n      <td>1605036970660462602</td>\n      <td>1522221998151061509</td>\n      <td>This time @TeamMessi Argentina ⚽❤✌ \\n\\n 📕 #Dex...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-20 03:06:48+00:00</td>\n      <td>en</td>\n      <td>[1058376110]</td>\n    </tr>\n    <tr>\n      <th>19958</th>\n      <td>#Qatar2022</td>\n      <td>1605036968588484609</td>\n      <td>1504760689650552863</td>\n      <td>The real GOAT 🐐 congrats LEO @TeamMessi😭🇦🇷 \\n\\...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-20 03:06:48+00:00</td>\n      <td>en</td>\n      <td>[1058376110]</td>\n    </tr>\n    <tr>\n      <th>19959</th>\n      <td>#Qatar2022</td>\n      <td>1605036951786074113</td>\n      <td>4896706619</td>\n      <td>#MessiCampeonMundial \\n#Qatar2022 \\n#Scaloneta...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2022-12-20 03:06:44+00:00</td>\n      <td>und</td>\n      <td>None</td>\n    </tr>\n  </tbody>\n</table>\n<p>39889 rows × 11 columns</p>\n</div>"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Displaying collected tweets\n",
    "tweets_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "                        id         username                created_at  \\\n0      1447603056359714817   24horasYucatan 2021-10-11 16:40:53+00:00   \n1      1485652252002066437       MarocV_Com 2022-01-24 16:34:46+00:00   \n2      1447604020462370822  24horasCampeche 2021-10-11 16:44:38+00:00   \n3      1055130038092345344      24horasqroo 2018-10-24 16:12:59+00:00   \n4                191106942       BVA_France 2010-09-15 16:36:24+00:00   \n...                    ...              ...                       ...   \n24738  1363653333479788549  Gabrielagladys5 2021-02-22 00:54:46+00:00   \n24739  1370074650038910983      dairy_truth 2021-03-11 18:10:43+00:00   \n24740  1246189682150367233       MilkedFilm 2020-04-03 21:36:08+00:00   \n24741  1522221998151061509  KishanSadhashiv 2022-05-05 14:31:08+00:00   \n24742           4896706619         Miroki03 2016-02-11 02:31:22+00:00   \n\n       followers_count  following_count  tweet_count  protected  verified  \n0                 3819              149        15217      False     False  \n1                  320                3         7737      False     False  \n2                 2473              110        13930      False     False  \n3                 6364             1187        76743      False     False  \n4                17606              605         7295      False     False  \n...                ...              ...          ...        ...       ...  \n24738               40              108         7815      False     False  \n24739             1278              428         3462      False     False  \n24740              793               52          111      False     False  \n24741                2               18          163      False     False  \n24742              647              639       119457      False     False  \n\n[49043 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>username</th>\n      <th>created_at</th>\n      <th>followers_count</th>\n      <th>following_count</th>\n      <th>tweet_count</th>\n      <th>protected</th>\n      <th>verified</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1447603056359714817</td>\n      <td>24horasYucatan</td>\n      <td>2021-10-11 16:40:53+00:00</td>\n      <td>3819</td>\n      <td>149</td>\n      <td>15217</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1485652252002066437</td>\n      <td>MarocV_Com</td>\n      <td>2022-01-24 16:34:46+00:00</td>\n      <td>320</td>\n      <td>3</td>\n      <td>7737</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1447604020462370822</td>\n      <td>24horasCampeche</td>\n      <td>2021-10-11 16:44:38+00:00</td>\n      <td>2473</td>\n      <td>110</td>\n      <td>13930</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1055130038092345344</td>\n      <td>24horasqroo</td>\n      <td>2018-10-24 16:12:59+00:00</td>\n      <td>6364</td>\n      <td>1187</td>\n      <td>76743</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>191106942</td>\n      <td>BVA_France</td>\n      <td>2010-09-15 16:36:24+00:00</td>\n      <td>17606</td>\n      <td>605</td>\n      <td>7295</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>24738</th>\n      <td>1363653333479788549</td>\n      <td>Gabrielagladys5</td>\n      <td>2021-02-22 00:54:46+00:00</td>\n      <td>40</td>\n      <td>108</td>\n      <td>7815</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>24739</th>\n      <td>1370074650038910983</td>\n      <td>dairy_truth</td>\n      <td>2021-03-11 18:10:43+00:00</td>\n      <td>1278</td>\n      <td>428</td>\n      <td>3462</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>24740</th>\n      <td>1246189682150367233</td>\n      <td>MilkedFilm</td>\n      <td>2020-04-03 21:36:08+00:00</td>\n      <td>793</td>\n      <td>52</td>\n      <td>111</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>24741</th>\n      <td>1522221998151061509</td>\n      <td>KishanSadhashiv</td>\n      <td>2022-05-05 14:31:08+00:00</td>\n      <td>2</td>\n      <td>18</td>\n      <td>163</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>24742</th>\n      <td>4896706619</td>\n      <td>Miroki03</td>\n      <td>2016-02-11 02:31:22+00:00</td>\n      <td>647</td>\n      <td>639</td>\n      <td>119457</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n<p>49043 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Displaying collected users\n",
    "users_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "# Saving collected data\n",
    "if not os.path.exists('./data/tweets.csv'):\n",
    "    tweets_df.to_csv(f'./data/tweets.csv', index=False)\n",
    "    users_df.to_csv(f'./data/users.csv', index=False)\n",
    "else:\n",
    "    tweets_df.to_csv(f'./data/tweets.csv', mode='a', index=False, header=False)\n",
    "    users_df.to_csv(f'./data/users.csv', mode='a', index=False, header=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}