passive_like.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "passive_like.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOQLFRl+ToLGYl6zotTWNL5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/VirtualGoat/Twitter-Data-Mining/blob/master/passive_like.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdZWP215wrbK",
        "colab_type": "code",
        "outputId": "4688b33c-c267-44a2-9c93-eaf9bd47682d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        }
      },
      "source": [
        "!pip install GetOldTweets3"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting GetOldTweets3\n",
            "  Downloading https://files.pythonhosted.org/packages/ed/f4/a00c2a7c90801abc875325bb5416ce9090ac86d06a00cc887131bd73ba45/GetOldTweets3-0.0.11-py3-none-any.whl\n",
            "Collecting pyquery>=1.2.10\n",
            "  Downloading https://files.pythonhosted.org/packages/78/43/95d42e386c61cb639d1a0b94f0c0b9f0b7d6b981ad3c043a836c8b5bc68b/pyquery-1.4.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: lxml>=3.5.0 in /usr/local/lib/python3.6/dist-packages (from GetOldTweets3) (4.2.6)\n",
            "Collecting cssselect>0.7.9\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/d4/3b5c17f00cce85b9a1e6f91096e1cc8e8ede2e1be8e96b87ce1ed09e92c5/cssselect-1.1.0-py2.py3-none-any.whl\n",
            "Installing collected packages: cssselect, pyquery, GetOldTweets3\n",
            "Successfully installed GetOldTweets3-0.0.11 cssselect-1.1.0 pyquery-1.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5uSSjakaw_Ez",
        "colab_type": "code",
        "outputId": "3b015b4a-1b18-477c-896b-0bc1ec057ae5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "#Accessing the data that has been stored. \n",
        "from google.colab import drive\n",
        "import pickle\n",
        "drive.mount('/content/drive')\n",
        "DATA_PATH = \"/content/drive/My Drive/Colab Notebooks/Internship/Tweet Data/New/Mumbai\"\n",
        "users4=open(DATA_PATH+'/passive.pickle','rb')\n",
        "users=pickle.load(users4)\n",
        "users4.close()\n",
        "print(len(users))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "8672\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r14cf6o-xH9Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "liked=set()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h7fqtRibxrBY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import urllib\n",
        "import re\n",
        "def get_user_ids_of_post_likes(post_id):\n",
        "    try:\n",
        "        json_data = urllib.request.urlopen('https://twitter.com/i/activity/favorited_popup?id=' + post_id).read()\n",
        "        json_data = json_data.decode('utf-8')\n",
        "        found_ids = re.findall(r'data-user-id=\\\\\"+\\d+', json_data)\n",
        "        unique_ids = list(set([re.findall(r'\\d+', match)[0] for match in found_ids]))\n",
        "        return unique_ids\n",
        "\n",
        "    except urllib.request.HTTPError:\n",
        "        return False\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "siVENXYqw1FR",
        "colab_type": "code",
        "outputId": "c3f38eed-893a-482a-a883-05aa6ebc24b9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "r=0\n",
        "for i in users:\n",
        "  try:\n",
        "    u=get_user_ids_of_post_likes(i.id)\n",
        "    for j in u:\n",
        "      liked.add(j)\n",
        "  except:\n",
        "    continue\n",
        "print(len(liked))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "513\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jwGON06Hy6QX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "file= '/content/drive/My Drive/New/Mumbai/passive2.pickle'\n",
        "with open(DATA_PATH+'/passive.pickle','wb') as f:\n",
        "    pickle.dump(non,f)\n",
        "f.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BO98LRGc3ZsZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tot=set()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bQZtMiAC_bRF",
        "colab_type": "code",
        "outputId": "554cbe84-5e60-46ed-83b1-9187685cd899",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "#Accessing the data that has been stored. \n",
        "from google.colab import drive\n",
        "tot1=set()\n",
        "import pickle\n",
        "drive.mount('/content/drive')\n",
        "DATA_PATH = \"/content/drive/My Drive/Colab Notebooks/Internship/Tweet Data/New/Thane\"\n",
        "users4=open(DATA_PATH+'/passive4.pickle','rb')\n",
        "users=pickle.load(users4)\n",
        "users4.close()\n",
        "for i in users:\n",
        "  tot.add(i)\n",
        "  tot1.add(i)\n",
        "print(len(tot))\n",
        "print(len(tot1))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "1258\n",
            "163\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GF22IfzUMK4J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tweepy\n",
        "\n",
        "consumer_key=''\n",
        "consumer_secret= ''\n",
        "access_token=''\n",
        "access_token_secret=''\n",
        "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
        "auth.set_access_token(access_token, access_token_secret)\n",
        "api = tweepy.API(auth,wait_on_rate_limit=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0YpxoOIgMy_2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "passive=set()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FO-WTWt-Lhdr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "for i in users:\n",
        "  try:\n",
        "    u = api.get_user(i)\n",
        "    passive.add(u.screen_name)\n",
        "  except:\n",
        "    continue"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KCVD1FEmaeS_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "non=set()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2gScgMxxYI2j",
        "colab_type": "code",
        "outputId": "362e8419-4798-4851-86f8-30445cf6a726",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "no=['decor','realt','digi','acre','resid','stock','group','article','project','commerc','brick','india','guru','new','market', \\\n",
        "    'enterp','vastu','astro','estate','mall','hous','Thane','sky','room','home','flat','design','interior','furniture','trend', \\\n",
        "    'pack','build','work','propert','vastu','gym','loan','luxury','bloomberg','business','direct','time',\\\n",
        "    'revenue','day','bank','agen','job','minis','corpo','tech', \\\n",
        "    'boss','nation','associat','develop','broth','invest','people','dna','media','break','bombay','mumbai','thane','lodha','hiranandani','world','estate','logis','team' \\\n",
        "    'consul','connec','school','tour','finan','asia','multi','apart','hospi','floor','repor']\n",
        "\n",
        "for i in passive:\n",
        "    if any(word in i.lower() for word in no) == False:\n",
        "        non.add(i)\n",
        "print(len(non))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "8066\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}