From 4c51220dc451688f3a0eb486541d5baf01b9f70f Mon Sep 17 00:00:00 2001 From: Daniel Schwarz Date: Tue, 16 May 2023 20:05:20 -0400 Subject: [PATCH 1/4] Removed followed_tags highlight feature (incompatible with HTML render) Also converted polls.py to use HTML rendering --- .vscode/launch.json | 24 ++++++++++++++++++++++++ requirements-dev.txt | 1 + toot/console.py | 10 ++++++++++ toot/debugger.py | 10 ++++++++++ toot/tui/app.py | 17 ----------------- toot/tui/poll.py | 12 ++++++++---- toot/tui/timeline.py | 1 - toot/tui/utils.py | 15 --------------- 8 files changed, 53 insertions(+), 37 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 toot/debugger.py diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..93f37d2e --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Remote Attach", + "type": "python", + "request": "attach", + "connect": { + "host": "localhost", + "port": 9000 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "justMyCode": false + } + ] +} \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index dfa5b15a..e3976a53 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ sphinx sphinx-autobuild twine wheel +debugpy diff --git a/toot/console.py b/toot/console.py index 5c84d97c..6492473e 100644 --- a/toot/console.py +++ b/toot/console.py @@ -10,6 +10,7 @@ from toot import config, commands, CLIENT_NAME, CLIENT_WEBSITE, __version__ from toot.exceptions import ApiError, ConsoleError from toot.output import print_out, print_err +from .debugger import initialize_debugger VISIBILITY_CHOICES = ["public", "unlisted", "private", "direct"] VISIBILITY_CHOICES_STR = ", ".join(f"'{v}'" for v in VISIBILITY_CHOICES) @@ -178,6 +179,12 @@ def editor(value): "action": 'store_true', "default": False, }), + (["--debugger"], { + "help": "launch with vscode debugpy", + "action": 'store_true', + "default": False, + }), + ] # Arguments added to commands which require authentication @@ -901,6 +908,9 @@ def run_command(app, user, name, args): def main(): + if "--debugger" in sys.argv: + initialize_debugger() + # Enable debug logging if --debug is in args if "--debug" in sys.argv: filename = os.getenv("TOOT_LOG_FILE") diff --git a/toot/debugger.py b/toot/debugger.py new file mode 100644 index 00000000..bc85d829 --- /dev/null +++ b/toot/debugger.py @@ -0,0 +1,10 @@ +def initialize_debugger(): + import multiprocessing + + if multiprocessing.current_process().pid > 1: + import debugpy + + debugpy.listen(("0.0.0.0", 9000)) + print("VSCode Debugger is ready to be attached on port 9000, press F5", flush=True) + debugpy.wait_for_client() + print("VSCode Debugger is now attached", flush=True) diff --git a/toot/tui/app.py b/toot/tui/app.py index b49dee23..6dfeb306 100644 --- a/toot/tui/app.py +++ b/toot/tui/app.py @@ -120,7 +120,6 @@ def __init__(self, app, user, args): def run(self): self.loop.set_alarm_in(0, lambda *args: self.async_load_instance()) self.loop.set_alarm_in(0, lambda *args: self.async_load_followed_accounts()) - self.loop.set_alarm_in(0, lambda *args: self.async_load_followed_tags()) self.loop.set_alarm_in(0, lambda *args: self.async_load_timeline( is_initial=True, timeline_name="home")) self.loop.run() @@ -316,22 +315,6 @@ def _done_accounts(accounts): self.run_in_thread(_load_accounts, done_callback=_done_accounts) - def async_load_followed_tags(self): - def _load_tag_list(): - try: - return api.followed_tags(self.app, self.user) - except ApiError: - # not supported by all Mastodon servers so fail silently if necessary - return [] - - def _done_tag_list(tags): - if len(tags) > 0: - self.followed_tags = [t["name"] for t in tags] - else: - self.followed_tags = [] - - self.run_in_thread(_load_tag_list, done_callback=_done_tag_list) - def refresh_footer(self, timeline): """Show status details in footer.""" status, index, count = timeline.get_focused_status_with_counts() diff --git a/toot/tui/poll.py b/toot/tui/poll.py index 81756af2..8ad649c1 100644 --- a/toot/tui/poll.py +++ b/toot/tui/poll.py @@ -2,9 +2,9 @@ from toot import api from toot.exceptions import ApiError -from toot.utils import format_content -from .utils import highlight_hashtags, parse_datetime +from .utils import parse_datetime from .widgets import Button, CheckBox, RadioButton +from .richtext import ContentParser class Poll(urwid.ListBox): @@ -85,8 +85,12 @@ def generate_poll_detail(self): def generate_contents(self, status): yield urwid.Divider() - for line in format_content(status.data["content"]): - yield urwid.Text(highlight_hashtags(line, set())) + + parser = ContentParser() + widgetlist = parser.html_to_widgets(status.data["content"]) + + for line in widgetlist: + yield (line) yield urwid.Divider() yield self.build_linebox(self.generate_poll_detail()) diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py index 3fb9b350..70192005 100644 --- a/toot/tui/timeline.py +++ b/toot/tui/timeline.py @@ -313,7 +313,6 @@ def remove_status(self, status): class StatusDetails(urwid.Pile): def __init__(self, timeline: Timeline, status: Optional[Status]): self.status = status - self.followed_tags = timeline.tui.followed_tags self.followed_accounts = timeline.tui.followed_accounts reblogged_by = status.author if status and status.reblog else None diff --git a/toot/tui/utils.py b/toot/tui/utils.py index 2f49362d..84cb7da6 100644 --- a/toot/tui/utils.py +++ b/toot/tui/utils.py @@ -81,21 +81,6 @@ def _gen(): return list(_gen()) -def highlight_hashtags(line, followed_tags, attr="hashtag", followed_attr="followed_hashtag"): - hline = [] - - for p in re.split(HASHTAG_PATTERN, line): - if p.startswith("#"): - if p[1:].lower() in (t.lower() for t in followed_tags): - hline.append((followed_attr, p)) - else: - hline.append((attr, p)) - else: - hline.append(p) - - return hline - - def show_media(paths): """ Attempt to open an image viewer to show given media files. From 5c6644b30cb5a9820f12fc3edc506cf3148a5f6c Mon Sep 17 00:00:00 2001 From: Daniel Schwarz Date: Fri, 26 May 2023 17:53:15 -0400 Subject: [PATCH 2/4] Login to servers that don't honor the uri spec for V1::Instance Pleroma, Akkoma, and other servers do not follow the Mastodon spec for the 'uri' attribute which specifies that it contains the domain name of the instance. Instead, they return a complete URI. As a workaround, we now detect this situation and parse out the domain from the URI when necessary. This fixes issue #347. Thanks to @laleanor for their patch and @rjp for ideas on how to make it work with GotoSocial and other servers --- toot/auth.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/toot/auth.py b/toot/auth.py index 0ee2bace..db34d9fc 100644 --- a/toot/auth.py +++ b/toot/auth.py @@ -7,6 +7,7 @@ from toot import api, config, DEFAULT_INSTANCE, User, App from toot.exceptions import ApiError, ConsoleError from toot.output import print_out +from urllib.parse import urlparse def register_app(domain, base_url): @@ -46,8 +47,20 @@ def get_instance_domain(base_url): f"running Mastodon version {instance['version']}" ) + # Pleroma and its forks return an actual URI here, rather than a + # domain name like Mastodon. This is contrary to the spec.¯ + # in that case, parse out the domain and return it. + + parsed_uri = urlparse(instance["uri"]) + + if parsed_uri.netloc: + # Pleroma, Akkoma, GotoSocial, etc. + return parsed_uri.netloc + else: + # Others including Mastodon servers + return parsed_uri.path + # NB: when updating to v2 instance endpoint, this field has been renamed to `domain` - return instance["uri"] def create_user(app, access_token): From f14653a576b790f2ed2de3e53299fc22a2b03e76 Mon Sep 17 00:00:00 2001 From: Daniel Schwarz Date: Sat, 27 May 2023 21:50:17 -0400 Subject: [PATCH 3/4] Anchor tags with class = hashtag or mention_hashtag are highlighted Previously this only worked for anchor tags with nested spans. Now it works for anchor tags with or without nested spans. --- toot/tui/richtext.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py index fc178be1..0cf326d9 100644 --- a/toot/tui/richtext.py +++ b/toot/tui/richtext.py @@ -238,6 +238,13 @@ def _a(self, tag) -> Tuple: attr = self.get_best_anchor_attr(attrib_list) + if attr == "a": + # didn't find an attribute to use + # in the child markup, so let's + # try the anchor tag's own attributes + + attr = self.get_urwid_attr_name(tag) + # hashtag anchors have a class of "mention hashtag" # or "hashtag" # we'll return style "class_mention_hashtag" From 33afe39e813fab999071516f2d49b4a6714887f9 Mon Sep 17 00:00:00 2001 From: Daniel Schwarz Date: Sat, 27 May 2023 21:51:41 -0400 Subject: [PATCH 4/4] URLEncode hrefs; fixes crash bug with href urls in foreign scripts --- .vscode/launch.json | 24 ------------------------ requirements-dev.txt | 1 - toot/console.py | 10 ---------- toot/debugger.py | 10 ---------- toot/tui/richtext.py | 3 +++ toot/tui/timeline.py | 3 ++- toot/utils/__init__.py | 12 ++++++++++++ 7 files changed, 17 insertions(+), 46 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100644 toot/debugger.py diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 93f37d2e..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Remote Attach", - "type": "python", - "request": "attach", - "connect": { - "host": "localhost", - "port": 9000 - }, - "pathMappings": [ - { - "localRoot": "${workspaceFolder}", - "remoteRoot": "." - } - ], - "justMyCode": false - } - ] -} \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index e3976a53..dfa5b15a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,3 @@ sphinx sphinx-autobuild twine wheel -debugpy diff --git a/toot/console.py b/toot/console.py index 6492473e..5c84d97c 100644 --- a/toot/console.py +++ b/toot/console.py @@ -10,7 +10,6 @@ from toot import config, commands, CLIENT_NAME, CLIENT_WEBSITE, __version__ from toot.exceptions import ApiError, ConsoleError from toot.output import print_out, print_err -from .debugger import initialize_debugger VISIBILITY_CHOICES = ["public", "unlisted", "private", "direct"] VISIBILITY_CHOICES_STR = ", ".join(f"'{v}'" for v in VISIBILITY_CHOICES) @@ -179,12 +178,6 @@ def editor(value): "action": 'store_true', "default": False, }), - (["--debugger"], { - "help": "launch with vscode debugpy", - "action": 'store_true', - "default": False, - }), - ] # Arguments added to commands which require authentication @@ -908,9 +901,6 @@ def run_command(app, user, name, args): def main(): - if "--debugger" in sys.argv: - initialize_debugger() - # Enable debug logging if --debug is in args if "--debug" in sys.argv: filename = os.getenv("TOOT_LOG_FILE") diff --git a/toot/debugger.py b/toot/debugger.py deleted file mode 100644 index bc85d829..00000000 --- a/toot/debugger.py +++ /dev/null @@ -1,10 +0,0 @@ -def initialize_debugger(): - import multiprocessing - - if multiprocessing.current_process().pid > 1: - import debugpy - - debugpy.listen(("0.0.0.0", 9000)) - print("VSCode Debugger is ready to be attached on port 9000, press F5", flush=True) - debugpy.wait_for_client() - print("VSCode Debugger is now attached", flush=True) diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py index 0cf326d9..175de59e 100644 --- a/toot/tui/richtext.py +++ b/toot/tui/richtext.py @@ -10,6 +10,7 @@ from bs4.element import NavigableString, Tag from urwidgets import TextEmbed, Hyperlink, parse_text from urwid.util import decompose_tagmarkup +from toot.utils import urlencode_url class ContentParser: @@ -232,6 +233,8 @@ def _a(self, tag) -> Tuple: if not attrib_list: attrib_list = [tag] if href: + # urlencode the path and query portions of the URL + href = urlencode_url(href) # use ASCII ETX (end of record) as a # delimiter between the title and the HREF title += f"\x03{href}" diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py index 70192005..d8e20194 100644 --- a/toot/tui/timeline.py +++ b/toot/tui/timeline.py @@ -14,6 +14,7 @@ from toot.tui import app from toot.tui.utils import time_ago from toot.utils.language import language_name +from toot.utils import urlencode_url from urwidgets import Hyperlink, TextEmbed, parse_text logger = logging.getLogger("toot") @@ -324,7 +325,7 @@ def linkify_content(self, text) -> urwid.Widget: TRANSFORM = { # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget re.compile(r'(https?://[^\s]+)'): - lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[1], "link"))), + lambda g: (len(g[1]), urwid.Filler(Hyperlink(urlencode_url(g[1]), "link", g[1]))), } markup_list = [] diff --git a/toot/utils/__init__.py b/toot/utils/__init__.py index e8103acf..026c861f 100644 --- a/toot/utils/__init__.py +++ b/toot/utils/__init__.py @@ -5,6 +5,7 @@ import tempfile import unicodedata import warnings +from urllib.parse import urlparse, quote, unquote, urlencode from bs4 import BeautifulSoup from typing import Dict @@ -81,6 +82,17 @@ def assert_domain_exists(domain): raise ConsoleError("Domain {} not found".format(domain)) +def urlencode_url(url): + parsed_url = urlparse(url) + + # unencode before encoding, to prevent double-urlencoding + encoded_path = quote(unquote(parsed_url.path), safe="-._~()'!*:@,;+&=/") + encoded_query = urlencode({k: quote(unquote(v), safe="-._~()'!*:@,;?/") for k, v in parsed_url.params}) + encoded_url = parsed_url._replace(path=encoded_path, params=encoded_query).geturl() + + return encoded_url + + EOF_KEY = "Ctrl-Z" if os.name == 'nt' else "Ctrl-D"