From ceb39f9070aee58919dbec6fb53659574f8e8338 Mon Sep 17 00:00:00 2001 From: Jordan Matelsky Date: Mon, 7 Aug 2023 15:02:29 -0400 Subject: [PATCH] Remove twitter :( (#83) * fix: Remove Twitter because it no longer works :( * v0.7.0 * fix: Add mastodon to utils --- CHANGELOG.md | 20 ++-- README.md | 8 +- docs/Customizing.md | 27 ++--- docs/example_library_usage.py | 2 - example-config.json | 8 +- goosepaper/__init__.py | 2 +- goosepaper/storyprovider/test_mastodon.py | 2 - goosepaper/storyprovider/twitter.py | 119 ---------------------- goosepaper/test_utils.py | 22 +++- goosepaper/util.py | 5 +- requirements.txt | 4 - setup.py | 2 +- 12 files changed, 49 insertions(+), 172 deletions(-) delete mode 100644 goosepaper/storyprovider/twitter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 36013d6..6c19d66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,18 @@ # CHANGELOG -### **0.?.?** (Unreleased) +### **0.7.0** (August 7 2023) -- Fixes - - Fixed a bug where the weather provider would not correctly render celsius unit marker +- Fixes + - Fixed a bug where the weather provider would not correctly render celsius unit marker + - Fixed the RSS story provider, +- Improvements + - added a Mastodon story provider (#82) + - Removed the Twitter story provider, which no longer works due to Twitter's API changes ### **0.6.0** (June 5 2022) - Improvements - - Enable writing a BytesIO object instead of creating a file. Good for cases where you're running goosepaper on unprivileged systems. - - Fixes - Fixed the old broken weather provider by switching to Open-Meteo. - No longer builds images on pull-request branches. @@ -25,13 +27,10 @@ ### **0.5.0** (January 14 2022) - Improvements - - RSS stories now "fall back" gracefully on just rendering the title, if the full body cannot be rendered. This is in contrast with the old behavior, in which the story would not be rendered at all. - RSS, Reddit, and Twitter story providers now support a `since_days_ago` argument in their `config` dictionaries that enables you to specify how many days ago to start the search for stories. Older stories will not be included. - Add support for multiple styles, using the `"styles" config option. Options are `"Academy"`, `"FifthAvenue"`, and `"Autumn"`. Previous style (before v0.4.0) was `Autumn`. - - Housekeeping - - Moved the story providers into their own submodule: Note that this may break backward compatibility if you import these directly. ### **0.4.0** (January 13 2022) @@ -39,13 +38,10 @@ > Multiple fixes and improvements - Fixes - - Changed some document name comparisons to case insensitive (prevent document overwrites, esp. for Windows users) - Switched upload to require named arguments rather than positional - Fixes the `limit` arg in the RSS provider, which was being ignored - - Improvements - - Improve typing support - Added more error handling for file and syntax handling - Change to using the `VissibleName` attribute in all cases rather than filename @@ -53,9 +49,7 @@ - Added more information on how to customize your goospaper in the docs, @kwillno (#54) - Adds the option to provide a global config (thanks @sedennial! #48) - Lots of new options to customize the upload and generation process (thanks @sedennial! #48) - - Housekeeping - - Fixes a bunch of flake8 errors and warnings to keep things tidy ### **0.3.1** (April 29 2021) diff --git a/README.md b/README.md index bed6806..d8b4c60 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ goosepaper is a utility that delivers a daily newspaper to your remarkable tablet. that's cute! -you can include RSS feeds, Twitter feeds, news articles, wikipedia articles-of-the-day, weather, and more. I read it when I wake up so that I can feel anxious without having to get my phone. +you can include RSS feeds, Mastodon feeds, news articles, wikipedia articles-of-the-day, weather, and more. I read it when I wake up so that I can feel anxious without having to get my phone. ## survey @@ -112,7 +112,7 @@ Check out [this example PDF](https://github.com/j6k4m8/goosepaper/blob/master/do ## existing story providers ([want to write your own?](https://github.com/j6k4m8/goosepaper/blob/master/CONTRIBUTING.md)) - [Wikipedia Top News / Current Events](https://github.com/j6k4m8/goosepaper/blob/master/goosepaper/storyprovider/wikipedia.py) -- [Tweets](https://github.com/j6k4m8/goosepaper/blob/master/goosepaper/storyprovider/twitter.py) +- [Mastodon Toots](https://github.com/j6k4m8/goosepaper/blob/master/goosepaper/storyprovider/mastodon.py) - [Weather](https://github.com/j6k4m8/goosepaper/blob/master/goosepaper/storyprovider/weather.py). These stories appear in the "ear" of the front page, just like a regular ol' newspaper - [RSS Feeds](https://github.com/j6k4m8/goosepaper/blob/master/goosepaper/storyprovider/rss.py) - [Reddit Subreddits](https://github.com/j6k4m8/goosepaper/blob/master/goosepaper/storyprovider/reddit.py) @@ -135,7 +135,9 @@ I do not think so, but it is a good question! ### may i use this to browse twitter? -yes you may! you can add a list of usernames to the feed generator and it will make a print-ready version of twitter. this is helpful for when you are on twitter on your laptop but wish you had Other Twitter as well, in print form. +~~yes you may! you can add a list of usernames to the feed generator and it will make a print-ready version of twitter. this is helpful for when you are on twitter on your laptop but wish you had Other Twitter as well, in print form.~~ + +no! twitter has changed and now no one can play nicely with them. sorry! it is sad! # You May Also Like... diff --git a/docs/Customizing.md b/docs/Customizing.md index 7c9560d..0acb238 100644 --- a/docs/Customizing.md +++ b/docs/Customizing.md @@ -17,13 +17,6 @@ As an example we give the config delivered as an example `example-config.json`: "F": true } }, - { - "provider": "twitter", - "config": { - "usernames": ["axios", "NPR"], - "limit_per": 8 - } - }, { "provider": "wikipedia_current_events", "config": {} @@ -91,7 +84,7 @@ For more information on the styles and to see a gallery of the different stylesh ## Stories and StoryProviders -Stories in a Goosepaper are created by a StoryProvider. You can think of a StoryProvider as a "source." So you might have Twitter stories (`TwitterStoryProvider`), some blog posts (`RSSFeedStoryProvider`), etc. +Stories in a Goosepaper are created by a StoryProvider. You can think of a StoryProvider as a "source." So you might have wikipedia stories (`WikipediaCurrentEventsStoryProvider`), some blog posts (`RSSFeedStoryProvider`), etc. This section aims to be a comprehensive list of all storyproviders and how to configure them. (This was the case at time of writing.) @@ -125,7 +118,6 @@ Right now, these are the storyproviders built into this repository: - [CustomText](#CustomText) - [Reddit](#Reddit) - [RSS](#RSS) -- [Twitter](#Twitter) - [Weather](#Weather) - [Wikipedia Current Events](#Wikipedia) @@ -192,21 +184,22 @@ Default limiting value is `5`. | `limit` | int | 5 | The number of stories to get. | | `since_days_ago` | int | None | If provided, filter stories by recency. | -### Twitter +### Mastodon ```json -"provider" : "twitter" +"provider" : "mastodon" ``` -Returns tweets from given users. +Returns toots from given users. #### Parameters: -| Parameter | Type | Default | Description | -| ---------------- | ---------------- | ------- | ----------------------------------------------------------- | -| `usernames` | str or list[str] | None | Twitter usernames to use. Can be a single username or list. | -| `limit` | int | 8 | The number of stories to get. | -| `since_days_ago` | int | None | If provided, filter stories by recency. | +| Parameter | Type | Default | Description | +| ---------------- | ---- | ------- | ----------------------------------------------------- | +| `username` | str | None | Mastodon username to use. | +| `limit` | int | 8 | The number of stories to get. | +| `since_days_ago` | int | None | If provided, filter stories by recency. | +| `server` | str | None | The server to use (e.g., "https://neuromatch.social") | ### Weather diff --git a/docs/example_library_usage.py b/docs/example_library_usage.py index 04a38a6..1d3c1a3 100644 --- a/docs/example_library_usage.py +++ b/docs/example_library_usage.py @@ -4,7 +4,6 @@ from goosepaper.goosepaper import Goosepaper from goosepaper.storyprovider.reddit import RedditHeadlineStoryProvider from goosepaper.storyprovider.rss import RSSFeedStoryProvider -from goosepaper.storyprovider.twitter import MultiTwitterStoryProvider from goosepaper.storyprovider.weather import OpenMeteoWeatherStoryProvider from goosepaper.storyprovider.wikipedia import WikipediaCurrentEventsStoryProvider from goosepaper.upload import upload @@ -21,7 +20,6 @@ OpenMeteoWeatherStoryProvider(lat=42.3601, lon=-71.0589, F=True), RSSFeedStoryProvider("https://www.npr.org/feed/", limit=5), RSSFeedStoryProvider("https://www.statnews.com/feed/", limit=2), - MultiTwitterStoryProvider(["reuters", "bbcWorld", "axios", "NPR"], limit_per=5), RedditHeadlineStoryProvider("news"), RedditHeadlineStoryProvider("todayilearned"), ] diff --git a/example-config.json b/example-config.json index cc458e4..7195405 100644 --- a/example-config.json +++ b/example-config.json @@ -10,13 +10,11 @@ } }, { - "provider": "twitter", + "provider": "mastodon", "config": { "since_days_ago": 0.5, - "usernames": [ - "axios", - "NPR" - ], + "username": "jordan", + "server": "https://neuromatch.social", "limit_per": 8 } }, diff --git a/goosepaper/__init__.py b/goosepaper/__init__.py index 952595e..073805c 100644 --- a/goosepaper/__init__.py +++ b/goosepaper/__init__.py @@ -1,3 +1,3 @@ from .goosepaper import Goosepaper # noqa -__version__ = "0.6.0" +__version__ = "0.7.0" diff --git a/goosepaper/storyprovider/test_mastodon.py b/goosepaper/storyprovider/test_mastodon.py index faf822e..8856d64 100644 --- a/goosepaper/storyprovider/test_mastodon.py +++ b/goosepaper/storyprovider/test_mastodon.py @@ -1,5 +1,3 @@ -# neuromatch.social/@jordan.rss - from .mastodon import MastodonStoryProvider diff --git a/goosepaper/storyprovider/twitter.py b/goosepaper/storyprovider/twitter.py deleted file mode 100644 index b83b75f..0000000 --- a/goosepaper/storyprovider/twitter.py +++ /dev/null @@ -1,119 +0,0 @@ -import datetime -import enum -import twint -from typing import List - -import pandas as pd - -from ..util import PlacementPreference, clean_text -from .storyprovider import StoryProvider -from ..story import Story - - -class TwitterStoryProviderPriorityMode(enum.Enum): - DEFAULT = 0 - RECENT = 1 - TOP = 2 - RATIO = 3 - - -class TwitterStoryProvider(StoryProvider): - def __init__( - self, - username: str, - limit: int = 5, - since_days_ago: int = None, - priority_mode: TwitterStoryProviderPriorityMode = TwitterStoryProviderPriorityMode.DEFAULT, - ) -> None: - """ - Create a new TwitterStoryProvider that reads from a username. - - Prioritizes stories based upon the TwitterStoryProviderPriorityMode. - """ - self.username = username - self.limit = limit - self.priority_mode = priority_mode - self._since = ( - datetime.datetime.now() - datetime.timedelta(days=since_days_ago) - if since_days_ago - else None - ) - - def get_stories(self, limit: int = 10, **kwargs) -> List[Story]: - """ - Get a list of stories. - - Here, the headline is the @username, and the body text is the tweet. - """ - - c = twint.Config() - c.Pandas = True - c.Search = f"from:{self.username}" - c.Limit = min(self.limit, limit) - c.Hide_output = True - - twint.run.Search(c) - df = twint.storage.panda.Tweets_df # type: ignore - stories = [] - for _, row in list(df.iterrows()): - date = pd.to_datetime(row.date) - if self._since is not None and date < self._since: - continue - stories.append( - Story( - headline=None, - body_text=clean_text(row.tweet), - byline=f"@{self.username} on Twitter at {date.strftime('%I:%M %p')}", - date=date, - placement_preference=PlacementPreference.SIDEBAR, - ) - ) - if len(stories) >= limit: - break - return stories - - -class MultiTwitterStoryProvider(StoryProvider): - def __init__( - self, - usernames: List[str], - limit_per: int = 5, - since_days_ago: int = None, - priority_mode: TwitterStoryProviderPriorityMode = TwitterStoryProviderPriorityMode.DEFAULT, - ) -> None: - """ - Create a new story provider that reads tweets from several users. - - Arguments: - usernames (List[str]): A list of twitter usernames - limit_per (int: 5): A maximum number of tweets to fetch from each - user in `usernames` - priority_mode (TwitterStoryProviderPriorityMode): Which priority - technique to use. Not currently implemented. - - """ - self.usernames = usernames - self.limit_per = limit_per - self.priority_mode = priority_mode - self._since_days_ago = since_days_ago - - def get_stories(self, limit: int = 42, **kwargs) -> List[Story]: - """ - Get a list of tweets where each tweet is a story. - - Arguments: - limit (int: 15): The maximum number of tweets to fetch - - Returns: - List[Story]: A list of tweets - - """ - stories = [] - for username in self.usernames: - stories.extend( - TwitterStoryProvider( - username, self.limit_per, self._since_days_ago, self.priority_mode - ).get_stories(limit=self.limit_per) - ) - stories = sorted(stories, key=lambda story: story.date) - return stories[:limit] diff --git a/goosepaper/test_utils.py b/goosepaper/test_utils.py index 37b109b..622498a 100644 --- a/goosepaper/test_utils.py +++ b/goosepaper/test_utils.py @@ -21,14 +21,32 @@ def test_clean_text(): def test_construct_story_providers_from_config_dict(): assert construct_story_providers_from_config_dict({}) == [] stories = construct_story_providers_from_config_dict( - {"stories": [{"provider": "twitter", "config": {"usernames": "j6m8"}}]} + { + "stories": [ + { + "provider": "mastodon", + "config": { + "server": "https://neuromatch.social", + "username": "j6m8", + "limit": 1, + }, + } + ] + } ) assert len(stories) == 1 stories = construct_story_providers_from_config_dict( { "stories": [ - {"provider": "twitter", "config": {"usernames": ["j6m8"]}}, + { + "provider": "mastodon", + "config": { + "server": "https://neuromatch.social", + "username": "j6m8", + "limit": 1, + }, + }, {"provider": "reddit", "config": {"subreddit": "worldnews"}}, ] } diff --git a/goosepaper/util.py b/goosepaper/util.py index 4f3ed7c..2981932 100644 --- a/goosepaper/util.py +++ b/goosepaper/util.py @@ -57,10 +57,9 @@ def load_config_file(filepath: str) -> dict: def construct_story_providers_from_config_dict(config: dict): - from goosepaper.storyprovider.rss import RSSFeedStoryProvider - from goosepaper.storyprovider.twitter import MultiTwitterStoryProvider from goosepaper.storyprovider.reddit import RedditHeadlineStoryProvider + from goosepaper.storyprovider.mastodon import MastodonStoryProvider from goosepaper.storyprovider.storyprovider import CustomTextStoryProvider from goosepaper.storyprovider.weather import OpenMeteoWeatherStoryProvider from goosepaper.storyprovider.wikipedia import WikipediaCurrentEventsStoryProvider @@ -68,9 +67,9 @@ def construct_story_providers_from_config_dict(config: dict): StoryProviderConfigNames = { "lorem": CustomTextStoryProvider, "text": CustomTextStoryProvider, - "twitter": MultiTwitterStoryProvider, "reddit": RedditHeadlineStoryProvider, "weather": OpenMeteoWeatherStoryProvider, + "mastodon": MastodonStoryProvider, "openmeteo_weather": OpenMeteoWeatherStoryProvider, "wikipedia_current_events": WikipediaCurrentEventsStoryProvider, "rss": RSSFeedStoryProvider, diff --git a/requirements.txt b/requirements.txt index f8370b7..5a64559 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,6 @@ requests # for rss feeds and reddit: feedparser -# for twitter stories: -git+https://github.com/twintproject/twint.git@origin/master#egg=twint -pandas - # for wikipedia stories: bs4 lxml diff --git a/setup.py b/setup.py index 17afef9..fc1a55f 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from codecs import open as copen from setuptools import setup, find_packages -__version__ = "0.6.0" +__version__ = "0.7.0" here = path.abspath(path.dirname(__file__))