Webscraping Bootcamp Submission (Fall 2018)

app.py

@@ -23,6 +23,8 @@ def eval(cmd, input=None):
         return wordOfTheDay.eval()
     elif cmd['service'] == 'MBTA':
         return MBTA.eval(cmd['args'])
+    elif cmd['service'] == 'H':
+        return headlines.eval(cmd['args'])
     else:
         return "ERROR 42: service not recognized"
 
@@ -40,6 +42,8 @@ def special(incoming):
         body = weather.special
     elif incoming.upper() == "MBTA":
         body = MBTA.special
+    elif incoming.upper() == 'HEADLINES':
+        body = headlines.special
     elif incoming.upper() == "DEMO":
         ## welcome/instructions
         body = 'Thanks for using Harvard Now!\n'
@@ -64,7 +68,7 @@ def response():
     if request.method == "GET":
         incoming = request.args.get("phrase")
 
-    if incoming is None: 
+    if incoming is None:
         resp = twilio.twiml.Response()
         resp.message(special("DEMO"))
         return str(resp)

data.py

@@ -270,4 +270,7 @@
     {'service': 'MBTA', 'args': {'pg': ['orange/ONST', 'green/north']}, 'tags': ['MBTA', 'SUBWAY', 'T', 'SCHEDULE', 'LINE', 'ORANGE', 'NORTH', 'STATION', 'GREEN']},
     {'service': 'MBTA', 'args': {'pg': ['blue/BGOV', 'green/gover']}, 'tags': ['MBTA', 'SUBWAY', 'T', 'SCHEDULE', 'LINE', 'BLUE', 'GOVERNMENT', 'CENTER', 'GREEN']},
     {'service': 'MBTA', 'args': {'pg': ['orange/OHAY', 'green/haecl']}, 'tags': ['MBTA', 'SUBWAY', 'T', 'SCHEDULE', 'LINE', 'ORANGE', 'HAYMARKET', 'GREEN']},
+    {'service': 'H', 'args': {'url': 'https://www.thecrimson.com/', 'publication': 'Crimson'}, 'tags': ['HEADLINES', 'CRIMSON', 'HARVARD']},
+    {'service': 'H', 'args': {'url': 'https://news.harvard.edu/gazette/', 'publication': 'Gazette'}, 'tags': ['HEADLINES', 'GAZETTE', 'HARVARD']},
+    {'service': 'H', 'args': {'url': 'https://hbr.org/', 'publication': 'HBR'}, 'tags': ['HEADLINES', 'HBR', 'HARVARD', 'BUSINESS', 'REVIEW']}
 ]

services/__init__.py

@@ -3,3 +3,4 @@
 from weather import weather
 from wordOfTheDay import wordOfTheDay
 from MBTA import MBTA
+from headlines import headlines

services/headlines/headlines.py

@@ -0,0 +1,27 @@
+import urllib2, urllib
+from bs4 import BeautifulSoup
+
+def headlines(url_string):
+    # String containing all headlines from main page
+    headlines = ''
+
+    # Parse all headlines
+    website = urllib2.urlopen(url_string)
+    soup = BeautifulSoup(website.read(), 'html.parser')
+    headline_tags = soup.find_all('a', href=True)
+
+    # Add all headlines
+    for tag in headline_tags:
+        headlines += tag.get_text() + '\n'
+
+    return headlines
+
+def makeSpecial():
+    s = 'To get the headlines from a particular publication, use the format \'headlines publication\'.'
+    return s
+
+## Default
+special = makeSpecial()
+
+def eval(cmd):
+    return "Today's headlines from The " + cmd['publication'] + ':\n' + headlines(cmd['url'])