mission_to_mars.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HW#13 Web Scraping and Document Databases\n",
    "## Mission to Mars\n",
    "### Due February 25, 2018\n",
    "### Step 1 - Web Scraping and Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Import dependencies\n",
    "from bs4 import BeautifulSoup\n",
    "from splinter import Browser\n",
    "import pandas as pd\n",
    "import requests\n",
    "import time\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NASA Mars News"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Retrieve page with the requests module\n",
    "browser = Browser('chrome', headless=False) \n",
    "url = 'https://mars.nasa.gov/news/'\n",
    "browser.visit(url)\n",
    "html=browser.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Create a BeautifulSoup object and parse html\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "#Extract latest news title and paragraph\n",
    "#print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NASA InSight Mission to Mars Arrives at Launch Site\n",
      "NASA's InSight spacecraft has arrived at Vandenberg Air Force Base in central California to begin final preparations for a launch this May.\n"
     ]
    }
   ],
   "source": [
    "news_title= soup.find('div', class_='content_title').get_text()\n",
    "print(news_title)\n",
    "news_paragraph=soup.find('div', class_='rollover_description_inner').get_text()\n",
    "print(news_paragraph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### JPL Mars Space Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Use splinter to retrieve featured image from JPL page\n",
    "executable_path = {'executable_path': '/usr/local/bin/chromedriver'} \n",
    "browser = Browser('chrome', **executable_path, headless=False)\n",
    "url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'\n",
    "browser.visit(url)\n",
    "\n",
    "#Click through the pages to reach the link containing the high res jpg image\n",
    "time.sleep(2)\n",
    "browser.click_link_by_partial_text('FULL IMAGE')\n",
    "time.sleep(2)\n",
    "browser.click_link_by_partial_text('more info')\n",
    "time.sleep(2)\n",
    "browser.click_link_by_partial_text('.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://photojournal.jpl.nasa.gov/jpeg/PIA18886.jpg\n"
     ]
    }
   ],
   "source": [
    "#Retrieve image url\n",
    "html=browser.html\n",
    "soup=BeautifulSoup(html,'html.parser')\n",
    "#print(soup.prettify())\n",
    "featured_image_url=soup.find('img').get('src')\n",
    "print(featured_image_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mars Weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sol 1978 (Feb 28, 2018), Sunny, high -14C/6F, low -78C/-108F, pressure at 7.27 hPa, daylight 05:37-17:25'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Scrape the Mars Weather Twitter page. Save latest weather tweet text as variable\n",
    "url='https://twitter.com/marswxreport?lang=en'\n",
    "html=requests.get(url)\n",
    "soup = BeautifulSoup(html.text, 'html.parser')\n",
    "#print(soup.prettify())\n",
    "mars_weather=soup.find_all(string=re.compile(\"Sol\"), class_=\"TweetTextSize TweetTextSize--normal js-tweet-text tweet-text\")[0].text\n",
    "mars_weather"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mars Facts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mars Planet Profile</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Equatorial Diameter:</th>\n",
       "      <td>6,792 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Polar Diameter:</th>\n",
       "      <td>6,752 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mass:</th>\n",
       "      <td>6.42 x 10^23 kg (10.7% Earth)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Moons:</th>\n",
       "      <td>2 (Phobos &amp; Deimos)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Orbit Distance:</th>\n",
       "      <td>227,943,824 km (1.52 AU)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Orbit Period:</th>\n",
       "      <td>687 days (1.9 years)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Surface Temperature:</th>\n",
       "      <td>-153 to 20 °C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>First Record:</th>\n",
       "      <td>2nd millennium BC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recorded By:</th>\n",
       "      <td>Egyptian astronomers</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   \n",
       "Mars Planet Profile                                \n",
       "Equatorial Diameter:                       6,792 km\n",
       "Polar Diameter:                            6,752 km\n",
       "Mass:                 6.42 x 10^23 kg (10.7% Earth)\n",
       "Moons:                          2 (Phobos & Deimos)\n",
       "Orbit Distance:            227,943,824 km (1.52 AU)\n",
       "Orbit Period:                  687 days (1.9 years)\n",
       "Surface Temperature:                  -153 to 20 °C\n",
       "First Record:                     2nd millennium BC\n",
       "Recorded By:                   Egyptian astronomers"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Scrape HTML Mars Planet Profile\n",
    "url='https://space-facts.com/mars/'\n",
    "table=pd.read_html(url)\n",
    "#table\n",
    "#type(table)\n",
    "table_df=table[0]\n",
    "table_df=table_df.rename(columns={0:'Mars Planet Profile', 1: ''})\n",
    "table_df.set_index('Mars Planet Profile', inplace=True)\n",
    "table_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Convert table to html table\n",
    "table=table_df.to_html()\n",
    "#table\n",
    "\n",
    "facts_table=table.replace('\\n','')\n",
    "#facts_table\n",
    "\n",
    "table_df.to_html('facts_table.html')\n",
    "#!open facts_table.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\n\\n\\n': '\\nMars Planet Profile\\n\\n', '\\nEquatorial Diameter:\\n6,792 km\\n': '\\nPolar Diameter:\\n6,752 km\\n', '\\nMass:\\n6.42 x 10^23 kg (10.7% Earth)\\n': '\\nMoons:\\n2 (Phobos & Deimos)\\n', '\\nOrbit Distance:\\n227,943,824 km (1.52 AU)\\n': '\\nOrbit Period:\\n687 days (1.9 years)\\n', '\\nSurface Temperature:\\n-153 to 20 °C\\n': '\\nFirst Record:\\n2nd millennium BC\\n'}\n"
     ]
    }
   ],
   "source": [
    "# facts_table.html.replace('/n',\"\")\n",
    "    #Strip items from html table\n",
    "soup=BeautifulSoup(open('facts_table.html'), 'html.parser')\n",
    "table_description=[]\n",
    "table_values={}\n",
    "for item in soup.table('tr'):\n",
    "    #print(item.text)\n",
    "    table_description.append(item.text.strip(\":\"))\n",
    "table_values=dict([(k,v) for k,v in zip (table_description[::2], table_description[1::2])])\n",
    "print (table_values)\n",
    "#print(table_description)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mars Hemispheres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Scrape Mars's hemispheres images from USGS Astrogeology site\n",
    "executable_path = {'executable_path': '/usr/local/bin/chromedriver'} \n",
    "browser = Browser('chrome', **executable_path, headless=False)\n",
    "base_url='https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'\n",
    "# landing_url ='https://astrogeology.usgs.gov'\n",
    "#url=\"https://astrogeology.usgs.gov/search/map/Mars/Viking/cerberus_enhanced\"\n",
    "browser.visit(base_url)\n",
    "\n",
    "html=browser.html\n",
    "soup = BeautifulSoup(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'title': 'Cerberus Hemisphere ', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/cerberus_enhanced.tif/full.jpg'}, {'title': 'Schiaparelli Hemisphere ', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/schiaparelli_enhanced.tif/full.jpg'}, {'title': 'Syrtis Major Hemisphere ', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/syrtis_major_enhanced.tif/full.jpg'}, {'title': 'Valles Marineris Hemisphere ', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/valles_marineris_enhanced.tif/full.jpg'}]\n"
     ]
    }
   ],
   "source": [
    "#Create a dictionary containing hemisphere titles and images urls\n",
    "hemisphere_image_urls=[]\n",
    "hemisphere_dict={'title':[], 'img_url':[]}\n",
    "x=soup.find_all('h3')\n",
    "#print(x)\n",
    "\n",
    "for i in x:\n",
    "    y=i.get_text()\n",
    "    title=y.strip('Enhanced')\n",
    "    browser.click_link_by_partial_text(y)\n",
    "    url=browser.find_link_by_partial_href('download')['href']\n",
    "    hemisphere_dict={'title':title, 'img_url':url}\n",
    "    hemisphere_image_urls.append(hemisphere_dict)\n",
    "    browser.visit(base_url)\n",
    "print(hemisphere_image_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}