script2_.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VfpZdZxrl8Yc",
        "outputId": "5a6a2434-0cc9-431a-9eeb-4e684ce11256"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Get:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease [1,581 B]\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]\n",
            "Get:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  Packages [518 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,266 kB]\n",
            "Hit:9 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,337 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,244 kB]\n",
            "Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,156 kB]\n",
            "Get:13 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease [18.1 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,037 kB]\n",
            "Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,000 kB]\n",
            "Hit:16 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Hit:17 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Get:18 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages [21.7 kB]\n",
            "Fetched 7,941 kB in 3s (2,969 kB/s)\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "30 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  apparmor chromium-browser libfuse3-3 liblzo2-2 libudev1 snapd squashfs-tools\n",
            "  systemd-hwe-hwdb udev\n",
            "Suggested packages:\n",
            "  apparmor-profiles-extra apparmor-utils fuse3 zenity | kdialog\n",
            "The following NEW packages will be installed:\n",
            "  apparmor chromium-browser chromium-chromedriver libfuse3-3 liblzo2-2 snapd\n",
            "  squashfs-tools systemd-hwe-hwdb udev\n",
            "The following packages will be upgraded:\n",
            "  libudev1\n",
            "1 upgraded, 9 newly installed, 0 to remove and 29 not upgraded.\n",
            "Need to get 26.4 MB of archives.\n",
            "After this operation, 116 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 apparmor amd64 3.0.4-2ubuntu2.2 [595 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblzo2-2 amd64 2.10-2build3 [53.7 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 squashfs-tools amd64 1:4.5-3build1 [159 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libudev1 amd64 249.11-0ubuntu3.10 [78.0 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 udev amd64 249.11-0ubuntu3.10 [1,557 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfuse3-3 amd64 3.10.5-1build1 [81.2 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 snapd amd64 2.58+22.04.1 [23.8 MB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 chromium-browser amd64 1:85.0.4183.83-0ubuntu2.22.04.1 [49.2 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 chromium-chromedriver amd64 1:85.0.4183.83-0ubuntu2.22.04.1 [2,308 B]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 systemd-hwe-hwdb all 249.11.3 [2,908 B]\n",
            "Fetched 26.4 MB in 1s (29.2 MB/s)\n",
            "Preconfiguring packages ...\n",
            "Selecting previously unselected package apparmor.\n",
            "(Reading database ... 120895 files and directories currently installed.)\n",
            "Preparing to unpack .../apparmor_3.0.4-2ubuntu2.2_amd64.deb ...\n",
            "Unpacking apparmor (3.0.4-2ubuntu2.2) ...\n",
            "Selecting previously unselected package liblzo2-2:amd64.\n",
            "Preparing to unpack .../liblzo2-2_2.10-2build3_amd64.deb ...\n",
            "Unpacking liblzo2-2:amd64 (2.10-2build3) ...\n",
            "Selecting previously unselected package squashfs-tools.\n",
            "Preparing to unpack .../squashfs-tools_1%3a4.5-3build1_amd64.deb ...\n",
            "Unpacking squashfs-tools (1:4.5-3build1) ...\n",
            "Preparing to unpack .../libudev1_249.11-0ubuntu3.10_amd64.deb ...\n",
            "Unpacking libudev1:amd64 (249.11-0ubuntu3.10) over (249.11-0ubuntu3.9) ...\n",
            "Setting up libudev1:amd64 (249.11-0ubuntu3.10) ...\n",
            "Selecting previously unselected package udev.\n",
            "(Reading database ... 121102 files and directories currently installed.)\n",
            "Preparing to unpack .../udev_249.11-0ubuntu3.10_amd64.deb ...\n",
            "Unpacking udev (249.11-0ubuntu3.10) ...\n",
            "Selecting previously unselected package libfuse3-3:amd64.\n",
            "Preparing to unpack .../libfuse3-3_3.10.5-1build1_amd64.deb ...\n",
            "Unpacking libfuse3-3:amd64 (3.10.5-1build1) ...\n",
            "Selecting previously unselected package snapd.\n",
            "Preparing to unpack .../snapd_2.58+22.04.1_amd64.deb ...\n",
            "Unpacking snapd (2.58+22.04.1) ...\n",
            "Setting up apparmor (3.0.4-2ubuntu2.2) ...\n",
            "Created symlink /etc/systemd/system/sysinit.target.wants/apparmor.service → /lib/systemd/system/apparmor.service.\n",
            "Setting up liblzo2-2:amd64 (2.10-2build3) ...\n",
            "Setting up squashfs-tools (1:4.5-3build1) ...\n",
            "Setting up udev (249.11-0ubuntu3.10) ...\n",
            "invoke-rc.d: could not determine current runlevel\n",
            "invoke-rc.d: policy-rc.d denied execution of start.\n",
            "Setting up libfuse3-3:amd64 (3.10.5-1build1) ...\n",
            "Setting up snapd (2.58+22.04.1) ...\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/snapd.aa-prompt-listener.service → /lib/systemd/system/snapd.aa-prompt-listener.service.\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/snapd.apparmor.service → /lib/systemd/system/snapd.apparmor.service.\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/snapd.autoimport.service → /lib/systemd/system/snapd.autoimport.service.\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/snapd.core-fixup.service → /lib/systemd/system/snapd.core-fixup.service.\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/snapd.recovery-chooser-trigger.service → /lib/systemd/system/snapd.recovery-chooser-trigger.service.\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/snapd.seeded.service → /lib/systemd/system/snapd.seeded.service.\n",
            "Created symlink /etc/systemd/system/cloud-final.service.wants/snapd.seeded.service → /lib/systemd/system/snapd.seeded.service.\n",
            "Unit /lib/systemd/system/snapd.seeded.service is added as a dependency to a non-existent unit cloud-final.service.\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/snapd.service → /lib/systemd/system/snapd.service.\n",
            "Created symlink /etc/systemd/system/timers.target.wants/snapd.snap-repair.timer → /lib/systemd/system/snapd.snap-repair.timer.\n",
            "Created symlink /etc/systemd/system/sockets.target.wants/snapd.socket → /lib/systemd/system/snapd.socket.\n",
            "Created symlink /etc/systemd/system/final.target.wants/snapd.system-shutdown.service → /lib/systemd/system/snapd.system-shutdown.service.\n",
            "Selecting previously unselected package chromium-browser.\n",
            "(Reading database ... 121335 files and directories currently installed.)\n",
            "Preparing to unpack .../chromium-browser_1%3a85.0.4183.83-0ubuntu2.22.04.1_amd64.deb ...\n",
            "=> Installing the chromium snap\n",
            "==> Checking connectivity with the snap store\n",
            "===> System doesn't have a working snapd, skipping\n",
            "Unpacking chromium-browser (1:85.0.4183.83-0ubuntu2.22.04.1) ...\n",
            "Selecting previously unselected package chromium-chromedriver.\n",
            "Preparing to unpack .../chromium-chromedriver_1%3a85.0.4183.83-0ubuntu2.22.04.1_amd64.deb ...\n",
            "Unpacking chromium-chromedriver (1:85.0.4183.83-0ubuntu2.22.04.1) ...\n",
            "Selecting previously unselected package systemd-hwe-hwdb.\n",
            "Preparing to unpack .../systemd-hwe-hwdb_249.11.3_all.deb ...\n",
            "Unpacking systemd-hwe-hwdb (249.11.3) ...\n",
            "Setting up systemd-hwe-hwdb (249.11.3) ...\n",
            "Setting up chromium-browser (1:85.0.4183.83-0ubuntu2.22.04.1) ...\n",
            "update-alternatives: using /usr/bin/chromium-browser to provide /usr/bin/x-www-browser (x-www-browser) in auto mode\n",
            "update-alternatives: using /usr/bin/chromium-browser to provide /usr/bin/gnome-www-browser (gnome-www-browser) in auto mode\n",
            "Setting up chromium-chromedriver (1:85.0.4183.83-0ubuntu2.22.04.1) ...\n",
            "Processing triggers for udev (249.11-0ubuntu3.10) ...\n",
            "Processing triggers for hicolor-icon-theme (0.17-2) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.1) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for dbus (1.12.20-2ubuntu4.1) ...\n",
            "Collecting selenium\n",
            "  Downloading selenium-4.13.0-py3-none-any.whl (9.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.5/9.5 MB\u001b[0m \u001b[31m63.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3[socks]<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from selenium) (2.0.5)\n",
            "Collecting trio~=0.17 (from selenium)\n",
            "  Downloading trio-0.22.2-py3-none-any.whl (400 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m400.2/400.2 kB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting trio-websocket~=0.9 (from selenium)\n",
            "  Downloading trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.10/dist-packages (from selenium) (2023.7.22)\n",
            "Requirement already satisfied: attrs>=20.1.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (23.1.0)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (2.4.0)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (3.4)\n",
            "Collecting outcome (from trio~=0.17->selenium)\n",
            "  Downloading outcome-1.2.0-py2.py3-none-any.whl (9.7 kB)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup>=1.0.0rc9 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.1.3)\n",
            "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium)\n",
            "  Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
            "Collecting h11<1,>=0.9.0 (from wsproto>=0.14->trio-websocket~=0.9->selenium)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: outcome, h11, wsproto, trio, trio-websocket, selenium\n",
            "Successfully installed h11-0.14.0 outcome-1.2.0 selenium-4.13.0 trio-0.22.2 trio-websocket-0.11.1 wsproto-1.2.0\n"
          ]
        }
      ],
      "source": [
        "!apt update\n",
        "!apt install chromium-chromedriver\n",
        "!pip install selenium\n",
        "from selenium import webdriver\n",
        "options = webdriver.ChromeOptions()\n",
        "options.add_argument('--headless')\n",
        "options.add_argument('--no-sandbox')\n",
        "options.add_argument('--disable-dev-shm-usage')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WceID4N50idP"
      },
      "outputs": [],
      "source": [
        "def getDiscrption(href):\n",
        "  driver_temp = webdriver.Chrome(options=options)\n",
        "  driver_temp.get(href);\n",
        "  time.sleep(5);\n",
        "  description_div = driver_temp.find_element(By.XPATH,'//*[@id=\"app\"]/div[1]/div[1]/div/main')\n",
        "  description = ''\n",
        "  for des in description_div.find_elements(By.CLASS_NAME,'content'):\n",
        "    description = description + des.text\n",
        "  driver_temp.quit()\n",
        "  # print(description);\n",
        "  return description;\n",
        "\n",
        "def getCountriesList():\n",
        "  driver_temp = webdriver.Chrome(options=options)\n",
        "  driver_temp.get('https://www.scholarshipportal.com/scholarships');\n",
        "  time.sleep(2);\n",
        "  list_coun = driver_temp.find_elements(By.XPATH,'//*[@id=\"app\"]/div[1]/div/div[1]/div/div/ul/li');\n",
        "  Main_List = []\n",
        "  for a in list_coun:\n",
        "    link = a.find_element(By.TAG_NAME,'a').get_attribute('href')\n",
        "    Main_List.append(link)\n",
        "  driver_temp.quit()\n",
        "  return Main_List\n",
        "\n",
        "def getNoOfPages(link):\n",
        "  driver_temp = webdriver.Chrome(options=options)\n",
        "  driver_temp.get(link);\n",
        "  time.sleep(2);\n",
        "  _coun = driver_temp.find_element(By.XPATH,'//*[@id=\"app\"]/div[1]/div/div[2]/div/h1');\n",
        "  length = int(int(_coun.text.split(' ')[0])/15);\n",
        "  driver_temp.quit()\n",
        "  return length"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "771_jAqrmG1t",
        "outputId": "923d7cc1-0e6e-4483-e302-a67b62e4aa3a"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "WebDriverException",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-84abe859256c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mnew_Data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# open it, go to a website, and get results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mc_link\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgetCountriesList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m   \u001b[0;31m# print(c_link)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m   \u001b[0mlength\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetNoOfPages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc_link\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-2-39ef97aa9925>\u001b[0m in \u001b[0;36mgetCountriesList\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgetCountriesList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m   \u001b[0mdriver_temp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mChrome\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m   \u001b[0mdriver_temp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'https://www.scholarshipportal.com/scholarships'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m   \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/selenium/webdriver/chrome/webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, executable_path, port, options, service_args, desired_capabilities, service_log_path, chrome_options, service, keep_alive)\u001b[0m\n\u001b[1;32m     78\u001b[0m             \u001b[0mservice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mService\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutable_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mport\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mservice_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mservice_log_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m         super().__init__(\n\u001b[0m\u001b[1;32m     81\u001b[0m             \u001b[0mDesiredCapabilities\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCHROME\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"browserName\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m             \u001b[0;34m\"goog\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/selenium/webdriver/chromium/webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, browser_name, vendor_prefix, port, options, service_args, desired_capabilities, service_log_path, service, keep_alive)\u001b[0m\n\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mservice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mservice\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0mcount\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massert_process_still_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_connectable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36massert_process_still_running\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0mreturn_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mreturn_code\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mWebDriverException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Service {self.path} unexpectedly exited. Status code was: {return_code}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mis_connectable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mWebDriverException\u001b[0m: Message: Service chromedriver unexpectedly exited. Status code was: 1\n"
          ]
        }
      ],
      "source": [
        "import time\n",
        "import pandas as pd\n",
        "from selenium.webdriver.common.by import By\n",
        "new_Data = []\n",
        "# open it, go to a website, and get results\n",
        "for c_link in getCountriesList():\n",
        "  # print(c_link)\n",
        "  length = getNoOfPages(c_link);\n",
        "  print('Total pages',length)\n",
        "  country = c_link.split('/')[-1]\n",
        "  driver = webdriver.Chrome(options=options)\n",
        "  URL= c_link+'?page='\n",
        "  count=0\n",
        "  new_df=pd.DataFrame()\n",
        "  for i in range(1,length+1):\n",
        "      driver.get(URL+ str(i))\n",
        "  # path= r'E:\\FYP\\chromedriver_win32\\chromedriver.exe'\n",
        "  # driver=webdriver.Chrome(executable_path=path)\n",
        "  #driver.get(\"https://www.scholarshipportal.com/scholarships?page=3\")\n",
        "\n",
        "      time.sleep(5)\n",
        "      sch_title=driver.find_elements(By.XPATH,\"//div[@class='md:w-3/4']//a//div[@class='mb-4']/h3\")\n",
        "      href=driver.find_elements(By.XPATH,\"//div[@class='md:w-3/4']//a\")\n",
        "      degree=driver.find_elements(By.XPATH,\"//div[@class='md:w-3/4']//a//div[2]/div[2]/span\")\n",
        "      date=driver.find_elements(By.XPATH,\"//div[@class='md:w-3/4']//a//div[2]/div[4]/span\")\n",
        "      # print(href)\n",
        "      r=1\n",
        "      templist = []\n",
        "      time.sleep(2)\n",
        "      #test=pd.DataFrame()\n",
        "\n",
        "      for title,href,degree,apply_date in zip(sch_title,href,degree,date):\n",
        "          # try:\n",
        "              try:\n",
        "                  sch_title=title.text\n",
        "              except Exception as ex:\n",
        "                  sch_title=\"-\"\n",
        "              try:\n",
        "                  date=apply_date.text\n",
        "              except Exception as ex:\n",
        "                  date=\"-\"\n",
        "              try:\n",
        "                  href=href.get_attribute('href')\n",
        "                  description = getDiscrption(href);\n",
        "              except Exception as ex:\n",
        "                  href=\"-\"\n",
        "              try:\n",
        "                  degree=degree.text\n",
        "              except Exception as ex:\n",
        "                  degree=\"-\"\n",
        "\n",
        "              count+=1\n",
        "              Table_dict={ 'Title': sch_title,\n",
        "                          'Links':href,\n",
        "                          'Country':country,\n",
        "                          'degree':degree,\n",
        "                          'Apply Date':date,\n",
        "                          'Description': description\n",
        "                          }\n",
        "\n",
        "              templist.append(Table_dict)\n",
        "              #df = pd.DataFrame(templist)\n",
        "              new_Data.append(Table_dict)\n",
        "              # new_df.append(Table_dict,ignore_index=True)\n",
        "              print(len(new_Data))\n",
        "              r =r+1\n",
        "\n",
        "          # if there are no more table data to scrape\n",
        "          # except :\n",
        "          #     break\n",
        "          #df.to_csv('studyportal.csv')\n",
        "\n",
        "      i=i+1\n",
        "      new_df = pd.DataFrame(new_Data)\n",
        "      new_df.to_csv('studyportal.csv')\n",
        "      #print(df)\n",
        "      #saving the dataframe to a csv\n",
        "\n",
        "  #print(df)\n",
        "  driver.close()\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Wm-gzgACQ2CA"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}