Addresses #468 Added bing news search (#489)

* Added bing news search

* Fixed Travis

Author: Remorax

app/scrapers/__init__.py

@@ -44,16 +44,6 @@ def feed_gen(query, engine, count=10, qtype=''):
     engine = old_names.get(engine, engine)
     if engine in ('quora', 'youtube'):
         urls = scrapers[engine].search_without_count(query)
-    elif engine in ('baidu', 'parsijoo', 'yahoo') and qtype == 'news':
-        urls = scrapers[engine].news_search(query, count, qtype)
-    elif engine == 'mojeek' and qtype == 'news':
-        urls = scrapers[engine].news_search_without_count(query)
-    elif engine in ('bing', 'parsijoo') and qtype == 'vid':
-        urls = scrapers[engine].video_search_without_count(query)
-    elif engine in ('bing', 'parsijoo') and qtype == 'isch':
-        urls = scrapers[engine].image_search_without_count(query)
-    elif engine in ('ask',) and qtype == 'vid':
-        urls = scrapers[engine].video_search(query, count, qtype)
     else:
         urls = scrapers[engine].search(query, count, qtype)
     return urls

app/scrapers/bing.py

@@ -10,11 +10,10 @@ def __init__(self):
         self.url = 'http://www.bing.com/search'
         self.videoURL = 'https://www.bing.com/videos/search'
         self.imageURL = 'https://www.bing.com/images/search'
+        self.newsURL = 'https://www.bing.com/news/search'
         self.defaultStart = 1
         self.startKey = 'first'
         self.name = 'bing'
-        self.videoKey = 'FORM'
-        self.imageKey = 'FORM'
 
     @staticmethod
     def parse_response(soup):
@@ -73,3 +72,26 @@ def parse_image_response(soup):
         print('Bing parsed: ' + str(urls))
 
         return urls
+
+    @staticmethod
+    def parse_news_response(soup):
+        """ Parses the reponse and return set of urls
+        Returns: urls (list)
+                [[Tile1,url1], [Title2, url2],..]
+        """
+        urls = []
+        for div in soup.findAll('div', {'class': 't_s'}):
+            link = div.find('a', {'class': 'title'})
+            url = link['href']
+            title = link.getText()
+            title = title.replace('\n', '').replace('  ', '')
+            desc = div.find('div', {'class': 'snippet'}).getText()
+            desc = desc.replace('\n', '').replace('  ', '')
+            url_entry = {'title': title,
+                         'link': url,
+                         'desc': desc}
+            urls.append(url_entry)
+
+        print('Bing parsed: ' + str(urls))
+
+        return urls

app/scrapers/generalized.py

@@ -19,49 +19,25 @@ class Scraper:
     }
 
     def __init__(self):
+        self.name = "general"
         pass
 
     def get_page(self, query, startIndex=0, qtype=''):
         """ Fetch the google search results page
         Returns : Results Page
         """
         url = self.url
-        if qtype == 'vid':
-            if self.name in ['yahoo']:
+        if qtype == 'vid' and self.name in ['yahoo', 'ask', 'parsijoo',
+                                                            'bing']:
                 url = self.videoURL
-            elif self.name in ['ask']:
-                url = self.videoURL
-                payload = {self.queryKey: query, self.startKey: startIndex}
-                response = requests.get(
-                    url, headers=self.headers, params=payload
-                )
-                return response
-            else:
-                url = self.url
-        elif qtype == 'isch':
-            if self.name in ['yahoo']:
+        elif qtype == 'isch' and self.name in ['yahoo', 'parsijoo', 'bing']:
                 url = self.imageURL
-            else:
-                url = self.url
-        elif qtype == 'news':
-            if self.name == 'baidu':
-                url = self.newsURL
-                payload = {'word': query, self.startKey: startIndex}
-                response = requests.get(
-                    url, headers=self.headers, params=payload
-                )
-                return response
-            elif self.name == 'parsijoo':
-                url = self.newsURL
-                payload = {self.queryKey: query, 'page': startIndex}
-                response = requests.get(
-                    url, headers=self.headers, params=payload
-                )
-                return response
-            elif self.name == 'yahoo':
-                url = self.newsURL
+        elif qtype == 'news' and self.name in ['baidu', 'parsijoo', 'mojeek', 'bing']:
+            url = self.newsURL
         payload = {self.queryKey: query, self.startKey: startIndex,
                    self.qtype: qtype}
+        if self.name == 'mojeek' and qtype == 'news':
+            payload['fmt'] = 'news'
         response = requests.get(url, headers=self.headers, params=payload)
         print(response.url)
         return response
@@ -98,16 +74,13 @@ def search(self, query, num_results, qtype=''):
 
     def call_appropriate_parser(self, qtype, soup):
         new_results = ''
-        if qtype == 'vid':
-            if self.name in ['yahoo']:
+        if qtype == 'vid' and self.name in ['yahoo', 'ask', 'parsijoo',
+                                                            'bing']:
                 new_results = self.parse_video_response(soup)
-            else:
-                new_results = self.parse_response(soup)
-        elif qtype == 'isch':
-            if self.name in ['yahoo']:
+        elif qtype == 'isch' and self.name in ['yahoo', 'parsijoo', 'bing']:
                 new_results = self.parse_image_response(soup)
-            else:
-                new_results = self.parse_response(soup)
+        elif qtype == 'news' and self.name in ['parsijoo', 'mojeek', 'baidu', 'bing']:
+                new_results = self.parse_news_response(soup)
         else:
             new_results = self.parse_response(soup)
         return new_results
@@ -123,95 +96,3 @@ def search_without_count(self, query):
         soup = BeautifulSoup(response.text, 'html.parser')
         urls = self.parse_response(soup)
         return urls
-
-    def video_search(self, query, num_results, qtype=''):
-        urls = []
-        current_start = self.defaultStart
-
-        while (len(urls) < num_results):
-            response = self.get_page(query, current_start, qtype)
-            soup = BeautifulSoup(response.text, 'html.parser')
-            if qtype == 'vid':
-                if self.name in ['yahoo', 'ask']:
-                    new_results = self.parse_video_response(soup)
-                else:
-                    new_results = self.parse_response(soup)
-            else:
-                new_results = self.parse_response(soup)
-            if new_results is None:
-                break
-            urls.extend(new_results)
-            current_start = self.next_start(current_start, new_results)
-        return urls[: num_results]
-
-    def video_search_without_count(self, query):
-        """
-            Search for the query and return set of urls
-            Returns: list
-        """
-        urls = []
-        if self.name in ['parsijoo']:
-            url = self.videoURL
-            payload = {self.queryKey: query}
-        elif self.name in ['bing']:
-            url = self.videoURL
-            payload = {self.queryKey: query, self.videoKey: 'HDRSC3'}
-        response = requests.get(url, headers=self.headers, params=payload)
-        soup = BeautifulSoup(response.text, 'html.parser')
-        urls = self.parse_video_response(soup)
-        if len(urls) == 0:
-            return "No video with this Keyword"
-        else:
-            return urls
-
-    def image_search_without_count(self, query):
-        """
-            Search for the query and return set of urls
-            Returns: list
-        """
-        urls = []
-        if self.name in ['parsijoo']:
-            url = self.imageURL
-            payload = {self.queryKey: query}
-        elif self.name in ['bing']:
-            url = self.imageURL
-            payload = {self.queryKey: query, self.imageKey: 'HDRSC2'}
-        response = requests.get(url, headers=self.headers, params=payload)
-        soup = BeautifulSoup(response.text, 'html.parser')
-        urls = self.parse_image_response(soup)
-        return urls
-
-    def news_search(self, query, num_results, qtype=''):
-        """
-            Search for the query and return set of urls
-            Returns: list
-        """
-        urls = []
-        if self.name == 'parsijoo':
-            current_start = self.newsStart
-        else:
-            current_start = self.defaultStart
-
-        while (len(urls) < num_results):
-            response = self.get_page(query, current_start, qtype)
-            soup = BeautifulSoup(response.text, 'html.parser')
-            new_results = self.parse_news_response(soup)
-            if new_results is None:
-                break
-            urls.extend(new_results)
-            current_start = self.next_start(current_start, new_results)
-        return urls[: num_results]
-
-    def news_search_without_count(self, query):
-        """
-            Search for the query and return set of urls
-            Returns: list
-        """
-        urls = []
-        if self.name == 'mojeek':
-            url = self.newsURL
-            payload = {self.queryKey: query, 'fmt': 'news'}
-        response = requests.get(url, headers=self.headers, params=payload)
-        soup = BeautifulSoup(response.text, 'html.parser')
-        urls = self.parse_news_response(soup)
-        return urls

test/test_bing.py

@@ -40,3 +40,17 @@ def test_parse_video_response():
         'link': link_video,
     }]
     assert resp == expected_resp
+
+
+def test_parse_news_response():
+    html_text = """<div class="t_s"><div class="t_t"><a class="title"
+        href="mock_url">mock_title</a></div><div class="snippet">
+        mock_desc</div></div>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = Bing().parse_news_response(dummy_soup)
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url',
+        'desc': u'mock_desc',
+    }]
+    assert resp == expected_resp