extract external links.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bc168f0c-5e64-4044-ad47-4274943e1d7f",
   "metadata": {
    "tags": []
   },
   "source": [
    "# How to extract internal and external links using Python\n",
    "In this tutorial, you will learn how to indentify and extract internal and external links from websites through web scraping in python and get help identify internal linking opportunities and improve SEO."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "83e9d734-a76e-440e-9e9a-5b6f4a926170",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "23beaea6-679f-44d3-a916-edfba25684f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# url \n",
    "url = \"https://www.coursera.org/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "d3db23b2-c0bf-4537-9b5e-07cd379b4920",
   "metadata": {},
   "outputs": [],
   "source": [
    "# send an https request and get http reponse\n",
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "77d42b2c-de3a-481a-8668-cdc8c580098d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "87d924e1-01b9-4f79-be45-f432a0e1a608",
   "metadata": {},
   "outputs": [],
   "source": [
    "html_page = bs(response.content, 'html.parser')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "374311ab-f184-4fe1-99c5-d6cccf6ef4d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_links = html_page.find_all('a', href=re.compile(r\"^['http', 'https']\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "8d8d1e9f-406b-44ba-a7e5-f5c9b1de802a",
   "metadata": {},
   "outputs": [],
   "source": [
    "links = {'link':[], 'category':[]}\n",
    "for link in all_links:\n",
    "    href = link['href']\n",
    "    if href:\n",
    "        if r'coursera.com' in href:\n",
    "            links['link'].append(href)\n",
    "            links['category'].append('internal')\n",
    "        if href[0] == \"#\":\n",
    "            links['link'].append(f\"{url}{href}\")\n",
    "            links['category'].append('internal')\n",
    "            \n",
    "        if href.split(\":\")[0] in ['https', 'http'] and not r\"coursera.com\" in href:\n",
    "            links['link'].append(href)\n",
    "            links['category'].append('external')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "9103e509-8f7a-4693-882b-d394955e05cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all links from the domain\n",
    "def get_all_link(url):\n",
    "    user_agent = {'User-Agent': 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36'}\n",
    "    # make http request\n",
    "    response = requests.get(url)\n",
    "    # parse html content\n",
    "    html_page = bs(response.content, 'html.parser')\n",
    "    # get all links\n",
    "    all_links = html_page.find_all('a')\n",
    "    \n",
    "    return all_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "1307500d-bd72-4a07-8b03-ec508cd5992a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_link(all_links, url, name):\n",
    "    \n",
    "    links = {'link':[], 'category':[]}\n",
    "    \n",
    "    for link in all_links:\n",
    "        href = link['href']\n",
    "        if href:\n",
    "            if name in href:\n",
    "                links['link'].append(href)\n",
    "                links['category'].append('internal')\n",
    "            if href[0] == \"#\":\n",
    "                links['link'].append(f\"{url}{href}\")\n",
    "                links['category'].append('internal')\n",
    "\n",
    "            if href.split(\":\")[0] in ['https', 'http'] and not name in href:\n",
    "                links['link'].append(href)\n",
    "                links['category'].append('external')\n",
    "    return links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "e1c605c3-ae95-4fb3-9278-5ecee4ee11bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://alibaba.com'"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "domain_name = r\"alibaba.com\"\n",
    "url = \"https://\"+domain_name\n",
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "405737ba-8a6d-45fd-85fb-19258c945265",
   "metadata": {},
   "outputs": [],
   "source": [
    "domain_name = r\"alibaba.com\"\n",
    "url = \"https://\"+domain_name\n",
    "all_links = get_all_link(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "2395b22a-a287-444d-bacb-9166606a3abc",
   "metadata": {},
   "outputs": [],
   "source": [
    "links = extract_link(all_links, url, domain_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "699142eb-7690-4b43-a8b9-5bc09d2b8d09",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "c1d5d208-ebf1-4bae-b222-45174810df5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>//www.alibaba.com/</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>//www.alibaba.com/</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>//passport.alibaba.com/icbu_login.htm?tracelog...</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>//accounts.alibaba.com/register/register.htm?t...</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>//message.alibaba.com?tracelog=2020NewHeader_h...</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>http://rule.alibaba.com/rule/detail/2041.htm</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>http://idinfo.zjamr.zj.gov.cn//bscx.do?method=...</td>\n",
       "      <td>external</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>//www.alibaba.com/trade/servlet/page/static/co...</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>http://www.beian.gov.cn/portal/registerSystemI...</td>\n",
       "      <td>external</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>http://beian.miit.gov.cn</td>\n",
       "      <td>external</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>116 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  link  category\n",
       "0                                   //www.alibaba.com/  internal\n",
       "1                                   //www.alibaba.com/  internal\n",
       "2    //passport.alibaba.com/icbu_login.htm?tracelog...  internal\n",
       "3    //accounts.alibaba.com/register/register.htm?t...  internal\n",
       "4    //message.alibaba.com?tracelog=2020NewHeader_h...  internal\n",
       "..                                                 ...       ...\n",
       "111       http://rule.alibaba.com/rule/detail/2041.htm  internal\n",
       "112  http://idinfo.zjamr.zj.gov.cn//bscx.do?method=...  external\n",
       "113  //www.alibaba.com/trade/servlet/page/static/co...  internal\n",
       "114  http://www.beian.gov.cn/portal/registerSystemI...  external\n",
       "115                           http://beian.miit.gov.cn  external\n",
       "\n",
       "[116 rows x 2 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6afe3aba-0b77-4d46-ad05-e381cfbffb2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "def get_broken_links(url):\n",
    "\n",
    "\t# Set root domain.\n",
    "\troot_domain = domain.com\n",
    "\t\n",
    "\t# Internal function for validating HTTP status code.\n",
    "\tdef _validate_url(url):\n",
    "\t\tr = requests.head(url)\n",
    "\t\tif r.status_code == 404:\n",
    "\t\t\tbroken_links.append(url)\n",
    "\t\t\t\n",
    "\t# Make request to URL.\t\t\n",
    "\tdata = requests.get(url).text\n",
    "\t\n",
    "\t# Parse HTML from request.\n",
    "\tsoup = BeautifulSoup(data, features=\"html.parser\")\n",
    "\t\n",
    "\t# Create a list containing all links with the root domain.\n",
    "\tlinks = [link.get(\"href\") for link in soup.find_all(\"a\") if f\"//{root_domain}\" in link.get(\"href\")]\n",
    "\t\n",
    "\t# Initialize list for broken links.\n",
    "\tbroken_links = []\n",
    "\t\n",
    "\t# Loop through links checking for 404 responses, and append to list.\n",
    "\twith ThreadPoolExecutor(max_workers=8) as executor:\n",
    "\t\texecutor.map(_validate_url, links)\n",
    "\t\t\n",
    "\treturn broken_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "d5f0f7c5-2f1c-4ed5-99c1-31c6f6840369",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "fd7eeac7-ba5f-45a5-bd19-f73330d1886a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>https://pages.ebay.com/help/policies/privacy-p...</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345</th>\n",
       "      <td>https://pages.ebay.com/help/account/cookies-we...</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>https://www.ebay.com/adchoice/ccpa</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>347</th>\n",
       "      <td>https://www.ebay.com/adchoice</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <td>https://seal.digicert.com/seals/popup/?tag=BIE...</td>\n",
       "      <td>internal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  link  category\n",
       "344  https://pages.ebay.com/help/policies/privacy-p...  internal\n",
       "345  https://pages.ebay.com/help/account/cookies-we...  internal\n",
       "346                 https://www.ebay.com/adchoice/ccpa  internal\n",
       "347                      https://www.ebay.com/adchoice  internal\n",
       "348  https://seal.digicert.com/seals/popup/?tag=BIE...  internal"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a38a7370-b7da-4568-b076-bd04cc0130a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}