{html}
", recovery_attempt=True) + else: + continue + else: + name = e.name + # if our HTML starts with a tag, but not a block tag + # the HTML is out of spec. Attempt a fix by wrapping the + # HTML with + if ( + first_tag + and not recovery_attempt + and name + not in ( + "p", + "pre", + "li", + "blockquote", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + ) # NOTE: update this list if Mastodon starts supporting more block tags + ): + return self.html_to_widgets(f"{html}
", recovery_attempt=True) + + # First, look for a custom tag handler method in this class + # If that fails, fall back to inline_tag_to_text handler + method = getattr(self, "_" + name, self.inline_tag_to_text) + markup = method(e) # either returns a Widget, or plain text + first_tag = False + + if not isinstance(markup, urwid.Widget): + # plaintext, so create a padded text widget + txt = self.text_to_widget("", markup) + markup = urwid.Padding( + txt, + align="left", + width=("relative", 100), + min_width=None, + ) + widgets.append(markup) + # separate top level widgets with a blank line + widgets.append(urwid.Divider(" ")) + return widgets[:-1] # but suppress the last blank line + + def inline_tag_to_text(self, tag) -> Tuple: + """Convert html tag to plain text with tag as attributes recursively""" + markups = self.process_inline_tag_children(tag) + if not markups: + return (tag.name, "") + return (tag.name, markups) + + def process_inline_tag_children(self, tag) -> List: + """Recursively retrieve all children + and convert to a list of markup text""" + markups = [] + for child in tag.children: + if isinstance(child, Tag): + method = getattr(self, "_" + child.name, self.inline_tag_to_text) + markup = method(child) + markups.append(markup) + else: + markups.append(child) + return markups + + def text_to_widget(self, attr, markup) -> TextEmbed: + TRANSFORM = { + # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget + re.compile(r"(^.+)\x03(.+$)"): lambda g: ( + len(g[1]), + urwid.Filler(Hyperlink(g[2], anchor_attr, g[1])), + ), + } + markup_list = [] + + for run in markup: + if isinstance(run, tuple): + txt, attr_list = decompose_tagmarkup(run) + # find anchor titles with an ETX separator followed by href + m = re.match(r"(^.+)\x03(.+$)", txt) + if m: + anchor_attr = self.get_best_anchor_attr(attr_list) + markup_list.append( + parse_text( + txt, + TRANSFORM, + lambda pattern, groups, span: TRANSFORM[pattern](groups), + ) + ) + else: + markup_list.append(run) + else: + markup_list.append(run) + + return TextEmbed(markup_list) + + def process_block_tag_children(self, tag) -> List[urwid.Widget]: + """Recursively retrieve all children + and convert to a list of widgets + any inline tags containing text will be + converted to Text widgets""" + + pre_widget_markups = [] + post_widget_markups = [] + child_widgets = [] + found_nested_widget = False + + for child in tag.children: + if isinstance(child, Tag): + # child is a nested tag; process using custom method + # or default to inline_tag_to_text + method = getattr(self, "_" + child.name, self.inline_tag_to_text) + result = method(child) + if isinstance(result, urwid.Widget): + found_nested_widget = True + child_widgets.append(result) + else: + if not found_nested_widget: + pre_widget_markups.append(result) + else: + post_widget_markups.append(result) + else: + # child is text; append to the appropriate markup list + if not found_nested_widget: + pre_widget_markups.append(child) + else: + post_widget_markups.append(child) + + widget_list = [] + if len(pre_widget_markups): + widget_list.append(self.text_to_widget(tag.name, pre_widget_markups)) + + if len(child_widgets): + widget_list += child_widgets + + if len(post_widget_markups): + widget_list.append(self.text_to_widget(tag.name, post_widget_markups)) + + return widget_list + + def get_urwid_attr_name(self, tag) -> str: + """Get the class name and translate to a + name suitable for use as an urwid + text attribute name""" + + if "class" in tag.attrs: + clss = tag.attrs["class"] + if len(clss) > 0: + style_name = "class_" + "_".join(clss) + # return the class name, only if we + # find it as a defined palette name + if style_name in self.palette_names: + return style_name + + # fallback to returning the tag name + return tag.name + + # Tag handlers start here. + # Tags not explicitly listed are "supported" by + # rendering as text. + # Inline tags return a list of marked up text for urwid.Text + # Block tags return urwid.Widget + + def basic_block_tag_handler(self, tag) -> urwid.Widget: + """default for block tags that need no special treatment""" + return urwid.Pile(self.process_block_tag_children(tag)) + + def get_best_anchor_attr(self, attrib_list) -> str: + if not attrib_list: + return "" + flat_al = list(flatten(attrib_list)) + + for a in flat_al[0]: + # ref: https://docs.joinmastodon.org/spec/activitypub/ + # these are the class names (translated to attrib names) + # that we can support for display + + try: + if a[0] in ["class_hashtag", "class_mention_hashtag", "class_mention"]: + return a[0] + except KeyError: + continue + + return "a" + + def _a(self, tag) -> Tuple: + """anchor tag handler""" + + markups = self.process_inline_tag_children(tag) + if not markups: + return (tag.name, "") + + href = tag.attrs["href"] + title, attrib_list = decompose_tagmarkup(markups) + if not attrib_list: + attrib_list = [tag] + if href: + # urlencode the path and query portions of the URL + href = urlencode_url(href) + # use ASCII ETX (end of record) as a + # delimiter between the title and the HREF + title += f"\x03{href}" + + attr = self.get_best_anchor_attr(attrib_list) + + if attr == "a": + # didn't find an attribute to use + # in the child markup, so let's + # try the anchor tag's own attributes + + attr = self.get_urwid_attr_name(tag) + + # hashtag anchors have a class of "mention hashtag" + # or "hashtag" + # we'll return style "class_mention_hashtag" + # or "class_hashtag" + # in that case; see corresponding palette entry + # in constants.py controlling hashtag highlighting + + return (attr, title) + + def _blockquote(self, tag) -> urwid.Widget: + widget_list = self.process_block_tag_children(tag) + blockquote_widget = urwid.LineBox( + urwid.Padding( + urwid.Pile(widget_list), + align="left", + width=("relative", 100), + min_width=None, + left=1, + right=1, + ), + tlcorner="", + tline="", + lline="│", + trcorner="", + blcorner="", + rline="", + bline="", + brcorner="", + ) + return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")]) + + def _br(self, tag) -> Tuple: + return ("br", "\n") + + def _em(self, tag) -> Tuple: + # to simplify the number of palette entries + # translate EM to I (italic) + markups = self.process_inline_tag_children(tag) + if not markups: + return ("i", "") + + # special case processing for bold and italic + for parent in tag.parents: + if parent.name == "b" or parent.name == "strong": + return ("bi", markups) + + return ("i", markups) + + def _ol(self, tag) -> urwid.Widget: + """ordered list tag handler""" + + widgets = [] + list_item_num = 1 + increment = -1 if tag.has_attr("reversed") else 1 + + # get ol start= attribute if present + if tag.has_attr("start") and len(tag.attrs["start"]) > 0: + try: + list_item_num = int(tag.attrs["start"]) + except ValueError: + pass + + for li in tag.find_all("li", recursive=False): + method = getattr(self, "_li", self.inline_tag_to_text) + markup = method(li) + + # li value= attribute will change the item number + # it also overrides any ol start= attribute + + if li.has_attr("value") and len(li.attrs["value"]) > 0: + try: + list_item_num = int(li.attrs["value"]) + except ValueError: + pass + + if not isinstance(markup, urwid.Widget): + txt = self.text_to_widget("li", [str(list_item_num), ". ", markup]) + # 1. foo, 2. bar, etc. + widgets.append(txt) + else: + txt = self.text_to_widget("li", [str(list_item_num), ". "]) + columns = urwid.Columns( + [txt, ("weight", 9999, markup)], dividechars=1, min_width=3 + ) + widgets.append(columns) + + list_item_num += increment + + return urwid.Pile(widgets) + + def _pre(self, tag) -> urwid.Widget: + # tag spec says that text should not wrap,
+ # but horizontal screen space is at a premium
+ # and we have no horizontal scroll bar, so allow
+ # wrapping.
+
+ widget_list = [urwid.Divider(" ")]
+ widget_list += self.process_block_tag_children(tag)
+
+ pre_widget = urwid.Padding(
+ urwid.Pile(widget_list),
+ align="left",
+ width=("relative", 100),
+ min_width=None,
+ left=1,
+ right=1,
+ )
+ return urwid.Pile([urwid.AttrMap(pre_widget, "pre")])
+
+ def _span(self, tag) -> Tuple:
+ markups = self.process_inline_tag_children(tag)
+
+ if not markups:
+ return (tag.name, "")
+
+ # span inherits its parent's class definition
+ # unless it has a specific class definition
+ # of its own
+
+ if "class" in tag.attrs:
+ # uncomment the following code to hide all HTML marked
+ # invisible (generally, the http:// prefix of URLs)
+ # could be a user preference, it's only advisable if
+ # the terminal supports OCS 8 hyperlinks (and that's not
+ # automatically detectable)
+
+ # if "invisible" in tag.attrs["class"]:
+ # return (tag.name, "")
+
+ style_name = self.get_urwid_attr_name(tag)
+
+ if style_name != "span":
+ # unique class name matches an entry in our palette
+ return (style_name, markups)
+
+ if tag.parent:
+ return (self.get_urwid_attr_name(tag.parent), markups)
+ else:
+ # fallback
+ return ("span", markups)
+
+ def _strong(self, tag) -> Tuple:
+ # to simplify the number of palette entries
+ # translate STRONG to B (bold)
+ markups = self.process_inline_tag_children(tag)
+ if not markups:
+ return ("b", "")
+
+ # special case processing for bold and italic
+ for parent in tag.parents:
+ if parent.name == "i" or parent.name == "em":
+ return ("bi", markups)
+
+ return ("b", markups)
+
+ def _ul(self, tag) -> urwid.Widget:
+ """unordered list tag handler"""
+
+ widgets = []
+
+ for li in tag.find_all("li", recursive=False):
+ method = getattr(self, "_li", self.inline_tag_to_text)
+ markup = method(li)
+
+ if not isinstance(markup, urwid.Widget):
+ txt = self.text_to_widget("li", ["\N{bullet} ", markup])
+ # * foo, * bar, etc.
+ widgets.append(txt)
+ else:
+ txt = self.text_to_widget("li", ["\N{bullet} "])
+ columns = urwid.Columns(
+ [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
+ )
+ widgets.append(columns)
+
+ return urwid.Pile(widgets)
+
+ # These tags are handled identically to others
+ # the only difference being the tag name used for
+ # urwid attribute mapping
+
+ _b = _strong
+
+ _div = basic_block_tag_handler
+
+ _i = _em
+
+ _li = basic_block_tag_handler
+
+ # Glitch-soc and Pleroma allow ... in content
+ # Mastodon (PR #23913) does not; header tags are converted to
+
+ _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
+
+ _p = basic_block_tag_handler
+
+
+def flatten(data):
+ if isinstance(data, tuple):
+ for x in data:
+ yield from flatten(x)
+ else:
+ yield data
diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py
index fb255c87..d8e20194 100644
--- a/toot/tui/timeline.py
+++ b/toot/tui/timeline.py
@@ -1,4 +1,5 @@
import logging
+import re
import sys
import urwid
import webbrowser
@@ -7,12 +8,14 @@
from .entities import Status
from .scroll import Scrollable, ScrollBar
-from .utils import highlight_hashtags, parse_datetime, highlight_keys
+from .utils import parse_datetime, highlight_keys
from .widgets import SelectableText, SelectableColumns
+from .richtext import ContentParser
from toot.tui import app
from toot.tui.utils import time_ago
-from toot.utils import format_content
from toot.utils.language import language_name
+from toot.utils import urlencode_url
+from urwidgets import Hyperlink, TextEmbed, parse_text
logger = logging.getLogger("toot")
@@ -311,7 +314,6 @@ def remove_status(self, status):
class StatusDetails(urwid.Pile):
def __init__(self, timeline: Timeline, status: Optional[Status]):
self.status = status
- self.followed_tags = timeline.tui.followed_tags
self.followed_accounts = timeline.tui.followed_accounts
reblogged_by = status.author if status and status.reblog else None
@@ -319,6 +321,18 @@ def __init__(self, timeline: Timeline, status: Optional[Status]):
if status else ())
return super().__init__(widget_list)
+ def linkify_content(self, text) -> urwid.Widget:
+ TRANSFORM = {
+ # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
+ re.compile(r'(https?://[^\s]+)'):
+ lambda g: (len(g[1]), urwid.Filler(Hyperlink(urlencode_url(g[1]), "link", g[1]))),
+ }
+ markup_list = []
+
+ markup_list.append(parse_text(text, TRANSFORM,
+ lambda pattern, groups, span: TRANSFORM[pattern](groups)))
+ return TextEmbed(markup_list, align='left')
+
def content_generator(self, status, reblogged_by):
if reblogged_by:
text = "♺ {} boosted".format(reblogged_by.display_name or reblogged_by.username)
@@ -341,8 +355,12 @@ def content_generator(self, status, reblogged_by):
yield ("pack", urwid.Text(("content_warning", "Marked as sensitive. Press S to view.")))
else:
content = status.original.translation if status.original.show_translation else status.data["content"]
- for line in format_content(content):
- yield ("pack", urwid.Text(highlight_hashtags(line, self.followed_tags)))
+
+ parser = ContentParser()
+ widgetlist = parser.html_to_widgets(content)
+
+ for line in widgetlist:
+ yield (line)
media = status.data["media_attachments"]
if media:
@@ -351,7 +369,7 @@ def content_generator(self, status, reblogged_by):
yield ("pack", urwid.Text([("bold", "Media attachment"), " (", m["type"], ")"]))
if m["description"]:
yield ("pack", urwid.Text(m["description"]))
- yield ("pack", urwid.Text(("link", m["url"])))
+ yield ("pack", self.linkify_content(m["url"]))
poll = status.original.data.get("poll")
if poll:
@@ -411,7 +429,7 @@ def card_generator(self, card):
if card["description"]:
yield urwid.Text(card["description"].strip())
yield urwid.Text("")
- yield urwid.Text(("link", card["url"]))
+ yield self.linkify_content(card["url"])
def poll_generator(self, poll):
for idx, option in enumerate(poll["options"]):
diff --git a/toot/tui/utils.py b/toot/tui/utils.py
index 2f49362d..84cb7da6 100644
--- a/toot/tui/utils.py
+++ b/toot/tui/utils.py
@@ -81,21 +81,6 @@ def _gen():
return list(_gen())
-def highlight_hashtags(line, followed_tags, attr="hashtag", followed_attr="followed_hashtag"):
- hline = []
-
- for p in re.split(HASHTAG_PATTERN, line):
- if p.startswith("#"):
- if p[1:].lower() in (t.lower() for t in followed_tags):
- hline.append((followed_attr, p))
- else:
- hline.append((attr, p))
- else:
- hline.append(p)
-
- return hline
-
-
def show_media(paths):
"""
Attempt to open an image viewer to show given media files.
diff --git a/toot/utils/__init__.py b/toot/utils/__init__.py
index e8103acf..026c861f 100644
--- a/toot/utils/__init__.py
+++ b/toot/utils/__init__.py
@@ -5,6 +5,7 @@
import tempfile
import unicodedata
import warnings
+from urllib.parse import urlparse, quote, unquote, urlencode
from bs4 import BeautifulSoup
from typing import Dict
@@ -81,6 +82,17 @@ def assert_domain_exists(domain):
raise ConsoleError("Domain {} not found".format(domain))
+def urlencode_url(url):
+ parsed_url = urlparse(url)
+
+ # unencode before encoding, to prevent double-urlencoding
+ encoded_path = quote(unquote(parsed_url.path), safe="-._~()'!*:@,;+&=/")
+ encoded_query = urlencode({k: quote(unquote(v), safe="-._~()'!*:@,;?/") for k, v in parsed_url.params})
+ encoded_url = parsed_url._replace(path=encoded_path, params=encoded_query).geturl()
+
+ return encoded_url
+
+
EOF_KEY = "Ctrl-Z" if os.name == 'nt' else "Ctrl-D"