Rich text support in console output

Makefile

@@ -8,7 +8,7 @@ publish :
 	twine upload dist/*.tar.gz dist/*.whl
 
 test:
-	pytest -v
+	pytest tests/*.py -v
 	flake8
 	vermin --target=3.6 --no-tips --violations --exclude-regex venv/.* .
 

requirements.txt

@@ -2,4 +2,5 @@ requests>=2.13,<3.0
 beautifulsoup4>=4.5.0,<5.0
 wcwidth>=0.1.7
 urwid>=2.0.0,<3.0
-
+urwidgets>=0.1,<0.2
+html2text>=2020.1.16

setup.py

@@ -38,6 +38,8 @@
         "beautifulsoup4>=4.5.0,<5.0",
         "wcwidth>=0.1.7",
         "urwid>=2.0.0,<3.0",
+        "urwidgets>=0.1,<0.2",
+        "html2text>=2020.1.16"
     ],
     entry_points={
         'console_scripts': [

tests/test_console.py

@@ -151,6 +151,74 @@ def test_timeline(mock_get, monkeypatch, capsys):
     assert err == ""
 
 
+@mock.patch('toot.http.get')
+def test_timeline_html_content(mock_get, monkeypatch, capsys):
+    mock_get.return_value = MockResponse([{
+        'id': '111111111111111111',
+        'account': {
+            'display_name': 'Frank Zappa 🎸',
+            'acct': 'fz'
+        },
+        'created_at': '2017-04-12T15:53:18.174Z',
+        'content': "<h2>HTML Render Test</h2><p><em>emphasized</em><br><u>underlined</u><br><strong>bold</strong><br><strong><em>bold and italic</em></strong><br><del>strikethrough</del><br>regular text</p><p>Code block:</p><pre><code>10 PRINT \"HELLO WORLD\"<br>20 GOTO 10<br></code></pre><blockquote><p>Something blockquoted here. The indentation is maintained as the text line wraps.</p></blockquote><ol><li>List item<ul><li>Nested item</li><li>Another nested </li></ul></li><li>Another list item. <ol><li>Something else nested</li><li>And a last nested</li></ol></li></ol><blockquote><p>Blockquote</p><ol><li>List in BQ</li><li>List item 2 in BQ</li></ol></blockquote><p><a href=\"https://babka.social/tags/hashtag\" class=\"mention hashtag\" rel=\"tag\">#<span>hashtag</span></a> <a href=\"https://babka.social/tags/test\" class=\"mention hashtag\" rel=\"tag\">#<span>test</span></a> <br><a href=\"https://a.com\" target=\"_blank\" rel=\"nofollow noopener noreferrer\"><span class=\"invisible\">https://</span><span class=\"\">a.com</span><span class=\"invisible\"></span></a> text after link</p>",
+        'reblog': None,
+        'in_reply_to_id': None,
+        'media_attachments': [],
+    }])
+
+    console.run_command(app, user, 'timeline', ['--once'])
+
+    mock_get.assert_called_once_with(app, user, '/api/v1/timelines/home', {'limit': 10})
+
+    out, err = capsys.readouterr()
+    lines = out.split("\n")  
+    reference = [
+        "────────────────────────────────────────────────────────────────────────────────────────────────────",
+        "Frank Zappa 🎸 @fz                                                              2017-04-12 15:53 UTC",
+        "",
+        "## HTML Render Test",
+        "",
+        " _emphasized_  ",
+        " _underlined_  ",
+        " **bold**  ",
+        " ** _bold and italic_**  ",
+        " ~~strikethrough~~  ",
+        "regular text",
+        "",
+        "Code block:",
+        "",
+        "    ",
+        "    10 PRINT \"HELLO WORLD\"  ",
+        "    20 GOTO 10  ",
+        "    ",
+        "> Something blockquoted here. The indentation is maintained as the text line wraps.",
+        "  1. List item",
+        "    • Nested item",
+        "    • Another nested ",
+        "  2. Another list item. ",
+        "    1. Something else nested",
+        "    2. And a last nested",
+        "",
+        "> Blockquote",
+        ">   1. List in BQ",
+        ">   2. List item 2 in BQ",
+        ">",
+        "",
+        "#hashtag #test  ",
+        "https://a.com text after link",
+        "",
+        "ID 111111111111111111   ",
+        "────────────────────────────────────────────────────────────────────────────────────────────────────",
+        "",
+    ]
+
+    assert len(lines) == len(reference)
+    for index, line in enumerate(lines):
+        assert line == reference[index], f"Line #{index}: Expected:\n{reference[index]}\nGot:\n{line}"
+
+    assert err == ""
+
+
 @mock.patch('toot.http.get')
 def test_timeline_with_re(mock_get, monkeypatch, capsys):
     mock_get.return_value = MockResponse([{
@@ -585,8 +653,6 @@ def test_notifications(mock_get, capsys):
         "────────────────────────────────────────────────────────────────────────────────────────────────────",
         "",
     ])
-
-
 @mock.patch('toot.http.get')
 def test_notifications_empty(mock_get, capsys):
     mock_get.return_value = MockResponse([])

toot/output.py

@@ -2,6 +2,7 @@
 import re
 import sys
 import textwrap
+import html2text
 
 from typing import List
 from wcwidth import wcswidth
@@ -272,6 +273,20 @@ def print_search_results(results):
 def print_status(status, width):
     reblog = status['reblog']
     content = reblog['content'] if reblog else status['content']
+
+    h2t = html2text.HTML2Text()
+
+    h2t.body_width = width
+    h2t.single_line_break = True
+    h2t.ignore_links = True
+    h2t.wrap_links = True
+    h2t.wrap_list_items = True
+    h2t.wrap_tables = True
+    h2t.unicode_snob = True
+    h2t.ul_item_mark = "\N{bullet}"
+
+    text_status = h2t.handle(content).strip()
+
     media_attachments = reblog['media_attachments'] if reblog else status['media_attachments']
     in_reply_to = status['in_reply_to_id']
     poll = reblog.get('poll') if reblog else status.get('poll')
@@ -294,7 +309,7 @@ def print_status(status, width):
     )
 
     print_out("")
-    print_html(content, width)
+    print_out(highlight_hashtags(text_status))
 
     if media_attachments:
         print_out("\nMedia:")

toot/tui/app.py

@@ -1,20 +1,22 @@
 import logging
 import urwid
+import html2text
 
 from concurrent.futures import ThreadPoolExecutor
 
 from toot import api, config, __version__
 from toot.console import get_default_visibility
 from toot.exceptions import ApiError
 
+
 from .compose import StatusComposer
 from .constants import PALETTE
 from .entities import Status
 from .overlays import ExceptionStackTrace, GotoMenu, Help, StatusSource, StatusLinks, StatusZoom
 from .overlays import StatusDeleteConfirmation, Account
 from .poll import Poll
 from .timeline import Timeline
-from .utils import parse_content_links, show_media, copy_to_clipboard
+from .utils import parse_content_links, show_media, copy_to_clipboard, parse_datetime
 
 logger = logging.getLogger(__name__)
 
@@ -119,7 +121,6 @@ def __init__(self, app, user, args):
     def run(self):
         self.loop.set_alarm_in(0, lambda *args: self.async_load_instance())
         self.loop.set_alarm_in(0, lambda *args: self.async_load_followed_accounts())
-        self.loop.set_alarm_in(0, lambda *args: self.async_load_followed_tags())
         self.loop.set_alarm_in(0, lambda *args: self.async_load_timeline(
             is_initial=True, timeline_name="home"))
         self.loop.run()
@@ -315,22 +316,6 @@ def _done_accounts(accounts):
 
         self.run_in_thread(_load_accounts, done_callback=_done_accounts)
 
-    def async_load_followed_tags(self):
-        def _load_tag_list():
-            try:
-                return api.followed_tags(self.app, self.user)
-            except ApiError:
-                # not supported by all Mastodon servers so fail silently if necessary
-                return []
-
-        def _done_tag_list(tags):
-            if len(tags) > 0:
-                self.followed_tags = [t["name"] for t in tags]
-            else:
-                self.followed_tags = []
-
-        self.run_in_thread(_load_tag_list, done_callback=_done_tag_list)
-
     def refresh_footer(self, timeline):
         """Show status details in footer."""
         status, index, count = timeline.get_focused_status_with_counts()
@@ -646,9 +631,26 @@ def _done(loop):
         return self.run_in_thread(_delete, done_callback=_done)
 
     def copy_status(self, status):
-        # TODO: copy a better version of status content
-        # including URLs
-        copy_to_clipboard(self.screen, status.original.data["content"])
+        h2t = html2text.HTML2Text()
+        h2t.body_width = 0  # nowrap
+        h2t.single_line_break = True
+        h2t.ignore_links = True
+        h2t.unicode_snob = True
+        h2t.ul_item_mark = "\N{bullet}"
+
+        time = parse_datetime(status.original.data['created_at'])
+        time = time.strftime('%Y-%m-%d %H:%M %Z')
+
+        text_status = (f"{status.original.data['url']}\n\n"
+            + (status.original.author.display_name or "")
+            + "\n"
+            + (status.original.author.account or "")
+            + "\n\n"
+            + h2t.handle(status.original.data["content"]).strip()
+            + "\n\n"
+            + f"Created at: {time}")
+
+        copy_to_clipboard(self.screen, text_status)
         self.footer.set_message(f"Status {status.original.id} copied")
 
     # --- Overlay handling -----------------------------------------------------

toot/tui/constants.py

@@ -37,7 +37,29 @@
     ('yellow_bold', 'yellow,bold', ''),
     ('red', 'dark red', ''),
     ('warning', 'light red', ''),
-    ('white_bold', 'white,bold', '')
+    ('white_bold', 'white,bold', ''),
+
+    # HTML tag styling
+    ('a', ',italics', ''),
+    # em tag is mapped to i
+    ('i', ',italics', ''),
+    # strong tag is mapped to b
+    ('b', ',bold', ''),
+    # special case for bold + italic nested tags
+    ('bi', ',bold,italics', ''),
+    ('u', ',underline', ''),
+    ('del', ',strikethrough', ''),
+    ('code', 'light gray, standout', ''),
+    ('pre', 'light gray, standout', ''),
+    ('blockquote', 'light gray', ''),
+    ('h1', ',bold', ''),
+    ('h2', ',bold', ''),
+    ('h3', ',bold', ''),
+    ('h4', ',bold', ''),
+    ('h5', ',bold', ''),
+    ('h6', ',bold', ''),
+    ('class_mention_hashtag', 'light cyan,bold', ''),
+    ('class_hashtag', 'light cyan,bold', ''),
 ]
 
 VISIBILITY_OPTIONS = [

toot/tui/overlays.py

@@ -4,10 +4,10 @@
 import webbrowser
 
 from toot import __version__
-from toot.utils import format_content
-from .utils import highlight_hashtags, highlight_keys
-from .widgets import Button, EditBox, SelectableText
 from toot import api
+from .utils import highlight_keys
+from .widgets import Button, EditBox, SelectableText
+from .richtext import ContentParser
 
 
 class StatusSource(urwid.Padding):
@@ -255,6 +255,8 @@ def setup_listbox(self):
         super().__init__(walker)
 
     def generate_contents(self, account, relationship=None, last_action=None):
+        parser = ContentParser()
+
         if self.last_action and not self.last_action.startswith("Confirm"):
             yield Button(f"Confirm {self.last_action}", on_press=take_action, user_data=self)
             yield Button("Cancel", on_press=cancel_action, user_data=self)
@@ -279,8 +281,10 @@ def generate_contents(self, account, relationship=None, last_action=None):
 
         if account["note"]:
             yield urwid.Divider()
-            for line in format_content(account["note"]):
-                yield urwid.Text(highlight_hashtags(line, followed_tags=set()))
+
+            widgetlist = parser.html_to_widgets(account["note"])
+            for line in widgetlist:
+                yield (line)
 
         yield urwid.Divider()
         yield urwid.Text(["ID: ", ("green", f"{account['id']}")])
@@ -312,8 +316,11 @@ def generate_contents(self, account, relationship=None, last_action=None):
                 name = field["name"].title()
                 yield urwid.Divider()
                 yield urwid.Text([("yellow", f"{name.rstrip(':')}"), ":"])
-                for line in format_content(field["value"]):
-                    yield urwid.Text(highlight_hashtags(line, followed_tags=set()))
+
+                widgetlist = parser.html_to_widgets(field["value"])
+                for line in widgetlist:
+                    yield (line)
+
                 if field["verified_at"]:
                     yield urwid.Text(("green", "✓ Verified"))
 

toot/tui/poll.py

@@ -2,9 +2,9 @@
 
 from toot import api
 from toot.exceptions import ApiError
-from toot.utils import format_content
-from .utils import highlight_hashtags, parse_datetime
+from .utils import parse_datetime
 from .widgets import Button, CheckBox, RadioButton
+from .richtext import ContentParser
 
 
 class Poll(urwid.ListBox):
@@ -85,8 +85,12 @@ def generate_poll_detail(self):
 
     def generate_contents(self, status):
         yield urwid.Divider()
-        for line in format_content(status.data["content"]):
-            yield urwid.Text(highlight_hashtags(line, set()))
+
+        parser = ContentParser()
+        widgetlist = parser.html_to_widgets(status.data["content"])
+
+        for line in widgetlist:
+            yield (line)
 
         yield urwid.Divider()
         yield self.build_linebox(self.generate_poll_detail())