Merge pull request #16 from mediacloud/feature-text-in-story-lists

add support for new `expanded` param in paged_articles (fix #15)

waybacknews/searchapi.py

@@ -118,11 +118,14 @@ def all_articles(self, query: str, start_date: dt.datetime, end_date: dt.datetim
                 more_pages = True
 
     def paged_articles(self, query: str, start_date: dt.datetime, end_date: dt.datetime,
-                       page_size: Optional[int] = 1000,  pagination_token: Optional[str] = None, **kwargs):
+                       page_size: Optional[int] = 1000,  expanded: bool = False,
+                       pagination_token: Optional[str] = None, **kwargs):
         """
         @return: one page of stories
         """
         params = {"q": "{} AND {}".format(query, self._date_query_clause(start_date, end_date))}
+        if expanded:
+            params['expanded'] = 1
         if pagination_token:
             params['resume'] = pagination_token
         params.update(kwargs)

waybacknews/tests/test_waybacknews.py

@@ -122,6 +122,19 @@ def test_paged_articles(self):
         page2_urls = [s['url'] for s in page2]
         assert page1_url1 not in page2_urls  # verify pages don't overlap
 
+    def test_paged_expanded_articles(self):
+        query = "biden"
+        start_date = dt.datetime(2023, 11, 25)
+        end_date = dt.datetime(2023, 11, 26)
+        page1, next_token1 = self._api.paged_articles(query, start_date, end_date)
+        for s in page1:
+            assert 'text_content' not in s
+        page2, next_token2 = self._api.paged_articles(query, start_date, end_date,
+                                                      pagination_token=next_token1, expanded=True)
+        for s in page2:
+            assert 'text_content' in s
+
+
     def test_top_sources(self):
         results = self._api.top_sources("coronavirus", dt.datetime(2022, 3, 1), dt.datetime(2022, 4, 1))
         assert len(results) > 0