From 80814ebdc970e78b007f7f5457457749fa9c7a6e Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 23 Mar 2023 22:47:56 -0400
Subject: [PATCH 01/55] Support for rendering a subset of HTML tags in status
 content

Code is adapted from GPL3-licensed muv by @seonon
https://github.com/seonon/muv
---
 .gitignore            |   2 +
 toot/tui/constants.py |  33 +++++-
 toot/tui/richtext.py  | 267 ++++++++++++++++++++++++++++++++++++++++++
 toot/tui/timeline.py  |  12 +-
 4 files changed, 309 insertions(+), 5 deletions(-)
 create mode 100644 toot/tui/richtext.py

diff --git a/.gitignore b/.gitignore
index bc647eb2..06bdc4a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 *.egg-info/
 *.pyc
 .pypirc
+.vscode
 /.cache/
 /.coverage
 /.env
@@ -14,3 +15,4 @@
 debug.log
 /pyrightconfig.json
 /book
+/venv
\ No newline at end of file
diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index e866e34a..13f201dd 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -37,7 +37,38 @@
     ('yellow_bold', 'yellow,bold', ''),
     ('red', 'dark red', ''),
     ('warning', 'light red', ''),
-    ('white_bold', 'white,bold', '')
+    ('white_bold', 'white,bold', ''),
+
+    # HTML tag styling
+
+    # note, anchor styling is often overridden
+    # by class names in Mastodon statuses
+    # so you won't see the italics.
+    ('a', ',italics', ''),
+    ('em', 'white,italics', ''),
+    ('i', 'white,italics', ''),
+
+    ('strong', 'white,bold', ''),
+    ('b', 'white,bold', ''),
+
+    ('u', 'white,underline', ''),
+
+    ('del', 'white, strikethrough', ''),
+
+    ('code', 'white, standout', ''),
+    ('pre', 'white, standout', ''),
+
+    ('blockquote', 'light gray', ''),
+
+    ('h1', 'yellow, bold', ''),
+    ('h2', 'dark red, bold', ''),
+    ('h3', 'yellow, bold', ''),
+    ('h4', 'yellow, bold', ''),
+    ('h5', 'yellow, bold', ''),
+    ('h6', 'yellow, bold', ''),
+
+    ('class_mention_hashtag', 'light cyan,bold', ''),
+
 ]
 
 VISIBILITY_OPTIONS = [
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
new file mode 100644
index 00000000..50315217
--- /dev/null
+++ b/toot/tui/richtext.py
@@ -0,0 +1,267 @@
+"""
+richtext
+"""
+from typing import List
+import urwid
+from bs4 import BeautifulSoup
+from bs4.element import NavigableString, Tag
+
+
+class ContentParser:
+    def __init__(self, config={}):
+        """Parse a limited subset of HTML and create urwid widgets."""
+        self.tag_to_method = {
+            "b": self.inline_tag_to_text,
+            "i": self.inline_tag_to_text,
+            "code": self.inline_tag_to_text,
+            "em": self.inline_tag_to_text,
+            "strong": self.inline_tag_to_text,
+            "del": self.inline_tag_to_text,
+        }
+
+    def html_to_widgets(self, html) -> List[urwid.Widget]:
+        """Convert html to urwid widgets"""
+        widgets: List[urwid.Widget] = []
+        soup = BeautifulSoup(html.replace('&apos;', "'"), "html.parser")
+        for e in soup.body or soup:
+            if isinstance(e, NavigableString):
+                continue
+            name = e.name
+            # get the custom method for the tag, defaulting to tag_to_text if none defined for this tag
+            method = self.tag_to_method.get(
+                name, getattr(self, "_" + name, self.inline_tag_to_text)
+            )
+
+            markup = method(e)  # either returns a Widget, or plain text
+            if not isinstance(markup, urwid.Widget):
+                # plaintext, so create a padded text widget
+                txt = urwid.Text(markup)
+                markup = urwid.Padding(
+                    txt,
+                    align="left",
+                    width=("relative", 100),
+                    min_width=None,
+                )
+            widgets.append(markup)
+        return widgets
+
+    def inline_tag_to_text(self, tag) -> list:
+        """Convert html tag to plain text with tag as attributes recursively"""
+        markups = self.process_inline_tag_children(tag)
+        if not markups:
+            return ""
+        return (tag.name, markups)
+
+    def process_inline_tag_children(self, tag) -> list:
+        markups = []
+        for child in tag.children:
+            if isinstance(child, Tag):
+                method = self.tag_to_method.get(
+                    child.name, getattr(self, "_" + child.name, self.inline_tag_to_text)
+                )
+                markup = method(child)
+                markups.append(markup)
+            else:
+                markups.append(child)
+        return markups
+
+    def process_block_tag_children(self, tag) -> List[urwid.Widget]:
+        pre_widget_markups = []
+        post_widget_markups = []
+        child_widgets = []
+        found_nested_widget = False
+
+        for child in tag.children:
+            if isinstance(child, Tag):
+                # child is a nested tag; process using custom method
+                # or default to inline_tag_to_text
+                method = self.tag_to_method.get(
+                    child.name, getattr(self, "_" + child.name, self.inline_tag_to_text)
+                )
+                result = method(child)
+                if isinstance(result, urwid.Widget):
+                    found_nested_widget = True
+                    child_widgets.append(result)
+                else:
+                    if not found_nested_widget:
+                        pre_widget_markups.append(result)
+                    else:
+                        post_widget_markups.append(result)
+            else:
+                # child is text; append to the appropriate markup list
+                if not found_nested_widget:
+                    pre_widget_markups.append(child)
+                else:
+                    post_widget_markups.append(child)
+
+        widget_list = []
+        if len(pre_widget_markups):
+            widget_list.append(urwid.Text((tag.name, pre_widget_markups)))
+
+        if len(child_widgets):
+            widget_list += child_widgets
+
+        if len(post_widget_markups):
+            widget_list.append(urwid.Text((tag.name, post_widget_markups)))
+
+        return widget_list
+
+    def get_style_name(self, tag) -> str:
+        # TODO: think about whitelisting allowed classes,
+        # or blacklisting classes we do not want.
+        # Classes to whitelist: "mention" "hashtag"
+        # used in anchor tags
+        # Classes to blacklist: "invisible" used in Akkoma
+        # anchor titles
+        style_name = tag.name
+        if "class" in tag.attrs:
+            clss = tag.attrs["class"]
+            if len(clss) > 0:
+                style_name = "class_" + "_".join(clss)
+        return style_name
+
+    # Tag handlers start here.
+    # Tags not explicitly listed are "supported" by
+    # rendering as text.
+    # Inline tags return a list of marked up text for urwid.Text
+    # Block tags return urwid.Widget
+
+
+    def basic_block_tag_handler(self, tag) -> urwid.Widget:
+        """default for block tags that need no special treatment"""
+        return urwid.Pile(self.process_block_tag_children(tag))
+
+    def _a(self, tag) -> list:
+        markups = self.process_inline_tag_children(tag)
+        if not markups:
+            return ""
+
+        # hashtag anchors have a class of "mention hashtag"
+        # we'll return style "class_mention_hashtag"
+        # in that case; set this up in constants.py
+        # to control highlighting of hashtags
+
+        return (self.get_style_name(tag), markups)
+
+    def _blockquote(self, tag) -> urwid.Widget:
+        widget_list = self.process_block_tag_children(tag)
+        blockquote_widget = urwid.LineBox(
+            urwid.Padding(
+                urwid.Pile(widget_list),
+                align="left",
+                width=("relative", 100),
+                min_width=None,
+                left=1,
+                right=1,
+            ),
+            tlcorner="",
+            tline="",
+            lline="│",
+            trcorner="",
+            blcorner="",
+            rline="",
+            bline="",
+            brcorner="",
+        )
+        return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")])
+
+
+    def _br(self, tag) -> list:
+        return (tag.name, ("br", "\n"))
+
+    _div = basic_block_tag_handler
+
+    _li = basic_block_tag_handler
+
+    # Glitch-soc and Pleroma allow <H1>...<H6> in content
+    # Mastodon (PR #23913) does not; header tags are converted to <STRONG>
+
+    _h1 = basic_block_tag_handler
+
+    _h2 = basic_block_tag_handler
+
+    _h3 = basic_block_tag_handler
+
+    _h4 = basic_block_tag_handler
+
+    _h5 = basic_block_tag_handler
+
+    _h6 = basic_block_tag_handler
+
+    def _ol(self, tag) -> urwid.Widget:
+        return self.list_widget(tag, ordered=True)
+
+    _p = basic_block_tag_handler
+
+    def _pre(self, tag) -> urwid.Widget:
+
+        # <PRE> tag spec says that text should not wrap,
+        # but horizontal screen space is at a premium
+        # and we have no horizontal scroll bar, so allow
+        # wrapping.
+
+        widget_list = [urwid.Divider(" ")]
+        widget_list += self.process_block_tag_children(tag)
+
+        pre_widget = urwid.Padding(
+            urwid.Pile(widget_list),
+            align="left",
+            width=("relative", 100),
+            min_width=None,
+            left=1,
+            right=1,
+        )
+        return urwid.Pile([urwid.AttrMap(pre_widget, "pre")])
+
+    def _span(self, tag) -> list:
+        markups = self.process_inline_tag_children(tag)
+
+        if not markups:
+            return ""
+
+        # span inherits its parent's class definition
+        # unless it has a specific class definition
+        # of its own
+
+        if "class" in tag.attrs:
+            style_name = self.get_style_name(tag)
+        elif tag.parent:
+            style_name = self.get_style_name(tag.parent)
+        else:
+            style_name = tag.name
+
+        return (style_name, markups)
+
+    def _ul(self, tag) -> urwid.Widget:
+        return self.list_widget(tag, ordered=False)
+
+    def list_widget(self, tag, ordered=False) -> urwid.Widget:
+        widgets = []
+        i = 1
+        for li in tag.find_all("li", recursive=False):
+            method = self.tag_to_method.get(
+                "li", getattr(self, "_li", self.inline_tag_to_text)
+            )
+            markup = method(li)
+
+            if not isinstance(markup, urwid.Widget):
+                if ordered:
+                    txt = urwid.Text(
+                        ("li", [str(i), ". ", markup])
+                    )  # 1. foo, 2. bar, etc.
+                else:
+                    txt = urwid.Text(("li", ["* ", markup]))  # * foo, * bar, etc.
+                widgets.append(txt)
+            else:
+                if ordered:
+                    txt = urwid.Text(("li", [str(i) + "."]))
+                else:
+                    txt = urwid.Text(("li", "*"))
+
+                columns = urwid.Columns(
+                    [txt, ("weight", 9999, markup)], dividechars=1, min_width=4
+                )
+                widgets.append(columns)
+            i += 1
+
+        return urwid.Pile(widgets)
diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py
index fb255c87..b102e294 100644
--- a/toot/tui/timeline.py
+++ b/toot/tui/timeline.py
@@ -7,11 +7,11 @@
 
 from .entities import Status
 from .scroll import Scrollable, ScrollBar
-from .utils import highlight_hashtags, parse_datetime, highlight_keys
+from .utils import parse_datetime, highlight_keys
 from .widgets import SelectableText, SelectableColumns
+from .richtext import ContentParser
 from toot.tui import app
 from toot.tui.utils import time_ago
-from toot.utils import format_content
 from toot.utils.language import language_name
 
 logger = logging.getLogger("toot")
@@ -341,8 +341,12 @@ def content_generator(self, status, reblogged_by):
             yield ("pack", urwid.Text(("content_warning", "Marked as sensitive. Press S to view.")))
         else:
             content = status.original.translation if status.original.show_translation else status.data["content"]
-            for line in format_content(content):
-                yield ("pack", urwid.Text(highlight_hashtags(line, self.followed_tags)))
+
+            parser = ContentParser()
+            widgetlist = parser.html_to_widgets(content)
+
+            for line in widgetlist:
+                yield (line)
 
             media = status.data["media_attachments"]
             if media:

From 3b4f46cb1fc2c5c1b778258e858e7d0982eb3d87 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Fri, 31 Mar 2023 22:27:03 -0400
Subject: [PATCH 02/55] comments and formatting

---
 toot/tui/richtext.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 50315217..fe05efe9 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -22,7 +22,7 @@ def __init__(self, config={}):
     def html_to_widgets(self, html) -> List[urwid.Widget]:
         """Convert html to urwid widgets"""
         widgets: List[urwid.Widget] = []
-        soup = BeautifulSoup(html.replace('&apos;', "'"), "html.parser")
+        soup = BeautifulSoup(html.replace("&apos;", "'"), "html.parser")
         for e in soup.body or soup:
             if isinstance(e, NavigableString):
                 continue
@@ -53,6 +53,8 @@ def inline_tag_to_text(self, tag) -> list:
         return (tag.name, markups)
 
     def process_inline_tag_children(self, tag) -> list:
+        """Recursively retrieve all children
+        and convert to a list of markup text"""
         markups = []
         for child in tag.children:
             if isinstance(child, Tag):
@@ -66,6 +68,11 @@ def process_inline_tag_children(self, tag) -> list:
         return markups
 
     def process_block_tag_children(self, tag) -> List[urwid.Widget]:
+        """Recursively retrieve all children
+        and convert to a list of widgets
+        any inline tags containing text will be
+        converted to Text widgets"""
+
         pre_widget_markups = []
         post_widget_markups = []
         child_widgets = []
@@ -107,6 +114,9 @@ def process_block_tag_children(self, tag) -> List[urwid.Widget]:
         return widget_list
 
     def get_style_name(self, tag) -> str:
+        """Get the class name and translate to a
+        name suitable for use as an urwid
+        text attribute name"""
         # TODO: think about whitelisting allowed classes,
         # or blacklisting classes we do not want.
         # Classes to whitelist: "mention" "hashtag"
@@ -126,7 +136,6 @@ def get_style_name(self, tag) -> str:
     # Inline tags return a list of marked up text for urwid.Text
     # Block tags return urwid.Widget
 
-
     def basic_block_tag_handler(self, tag) -> urwid.Widget:
         """default for block tags that need no special treatment"""
         return urwid.Pile(self.process_block_tag_children(tag))
@@ -165,7 +174,6 @@ def _blockquote(self, tag) -> urwid.Widget:
         )
         return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")])
 
-
     def _br(self, tag) -> list:
         return (tag.name, ("br", "\n"))
 
@@ -176,17 +184,7 @@ def _br(self, tag) -> list:
     # Glitch-soc and Pleroma allow <H1>...<H6> in content
     # Mastodon (PR #23913) does not; header tags are converted to <STRONG>
 
-    _h1 = basic_block_tag_handler
-
-    _h2 = basic_block_tag_handler
-
-    _h3 = basic_block_tag_handler
-
-    _h4 = basic_block_tag_handler
-
-    _h5 = basic_block_tag_handler
-
-    _h6 = basic_block_tag_handler
+    _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
 
     def _ol(self, tag) -> urwid.Widget:
         return self.list_widget(tag, ordered=True)
@@ -236,6 +234,8 @@ def _ul(self, tag) -> urwid.Widget:
         return self.list_widget(tag, ordered=False)
 
     def list_widget(self, tag, ordered=False) -> urwid.Widget:
+        """common logic for ordered and unordered list rendering
+        as urwid widgets"""
         widgets = []
         i = 1
         for li in tag.find_all("li", recursive=False):

From 717a0e459521682d6a51784d6f59264c926db382 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Fri, 31 Mar 2023 23:50:36 -0400
Subject: [PATCH 03/55] Removed unneeded import

---
 toot/commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/toot/commands.py b/toot/commands.py
index b17cf72b..58391dca 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -8,7 +8,7 @@
 from toot.auth import login_interactive, login_browser_interactive, create_app_interactive
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import (print_lists, print_out, print_instance, print_account, print_acct_list,
-                         print_search_results, print_table, print_timeline, print_notifications,
+                         print_search_results, print_timeline, print_notifications,
                          print_tag_list, print_list_accounts)
 from toot.tui.utils import parse_datetime
 from toot.utils import args_get_instance, delete_tmp_status_file, editor_input, multiline_input, EOF_KEY

From 479907bf42fcf0994f976070ba6db326169b3015 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 04/55] Support for rendering a subset of HTML tags in status
 content

Code is adapted from GPL3-licensed muv by @seonon
https://github.com/seonon/muv
---
 toot/tui/richtext.py | 31 ++++++++-----------------------
 1 file changed, 8 insertions(+), 23 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index fe05efe9..d361f909 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -10,14 +10,6 @@
 class ContentParser:
     def __init__(self, config={}):
         """Parse a limited subset of HTML and create urwid widgets."""
-        self.tag_to_method = {
-            "b": self.inline_tag_to_text,
-            "i": self.inline_tag_to_text,
-            "code": self.inline_tag_to_text,
-            "em": self.inline_tag_to_text,
-            "strong": self.inline_tag_to_text,
-            "del": self.inline_tag_to_text,
-        }
 
     def html_to_widgets(self, html) -> List[urwid.Widget]:
         """Convert html to urwid widgets"""
@@ -27,12 +19,11 @@ def html_to_widgets(self, html) -> List[urwid.Widget]:
             if isinstance(e, NavigableString):
                 continue
             name = e.name
-            # get the custom method for the tag, defaulting to tag_to_text if none defined for this tag
-            method = self.tag_to_method.get(
-                name, getattr(self, "_" + name, self.inline_tag_to_text)
-            )
-
+            # First, look for a custom tag handler method in this class
+            # If that fails, fall back to inline_tag_to_text handler
+            method = getattr(self, "_" + name, self.inline_tag_to_text)
             markup = method(e)  # either returns a Widget, or plain text
+
             if not isinstance(markup, urwid.Widget):
                 # plaintext, so create a padded text widget
                 txt = urwid.Text(markup)
@@ -58,9 +49,7 @@ def process_inline_tag_children(self, tag) -> list:
         markups = []
         for child in tag.children:
             if isinstance(child, Tag):
-                method = self.tag_to_method.get(
-                    child.name, getattr(self, "_" + child.name, self.inline_tag_to_text)
-                )
+                method = getattr(self, "_" + child.name, self.inline_tag_to_text)
                 markup = method(child)
                 markups.append(markup)
             else:
@@ -82,9 +71,7 @@ def process_block_tag_children(self, tag) -> List[urwid.Widget]:
             if isinstance(child, Tag):
                 # child is a nested tag; process using custom method
                 # or default to inline_tag_to_text
-                method = self.tag_to_method.get(
-                    child.name, getattr(self, "_" + child.name, self.inline_tag_to_text)
-                )
+                method = getattr(self, "_" + child.name, self.inline_tag_to_text)
                 result = method(child)
                 if isinstance(result, urwid.Widget):
                     found_nested_widget = True
@@ -182,7 +169,7 @@ def _br(self, tag) -> list:
     _li = basic_block_tag_handler
 
     # Glitch-soc and Pleroma allow <H1>...<H6> in content
-    # Mastodon (PR #23913) does not; header tags are converted to <STRONG>
+    # Mastodon (PR #23913) does not; header tags are converted to <P><STRONG></STRONG></P>
 
     _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
 
@@ -239,9 +226,7 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
         widgets = []
         i = 1
         for li in tag.find_all("li", recursive=False):
-            method = self.tag_to_method.get(
-                "li", getattr(self, "_li", self.inline_tag_to_text)
-            )
+            method = getattr(self, "_li", self.inline_tag_to_text)
             markup = method(li)
 
             if not isinstance(markup, urwid.Widget):

From 4ceb3e5ca1c75e235f6bc3c316e9d6e6a234b841 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 05/55] Ignore warning W503

see: https://www.flake8rules.com/rules/W503.html for justification
---
 .flake8 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.flake8 b/.flake8
index 21fd7bd0..6efbecd1 100644
--- a/.flake8
+++ b/.flake8
@@ -1,4 +1,4 @@
 [flake8]
 exclude=build,tests,tmp,venv,toot/tui/scroll.py
-ignore=E128
+ignore=E128,W503
 max-line-length=120

From eb8033ce4648e8bc666fdf6b1aaf044579cd6384 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 06/55] add get_lists method

---
 toot/api.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/toot/api.py b/toot/api.py
index f1beecee..7752af42 100644
--- a/toot/api.py
+++ b/toot/api.py
@@ -519,3 +519,8 @@ def clear_notifications(app, user):
 def get_instance(base_url):
     url = f"{base_url}/api/v1/instance"
     return http.anon_get(url).json()
+
+
+def get_lists(app, user):
+    path = "/api/v1/lists"
+    return _get_response_list(app, user, path)

From 524115e2f506c60e0476efadf816cb57b5c1dc44 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 07/55] Make button widget unicode-aware (spacing)

---
 toot/tui/widgets.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/toot/tui/widgets.py b/toot/tui/widgets.py
index 6f46fb33..f2ae4b8f 100644
--- a/toot/tui/widgets.py
+++ b/toot/tui/widgets.py
@@ -1,4 +1,5 @@
 import urwid
+from wcwidth import wcswidth
 
 
 class Clickable:
@@ -40,12 +41,12 @@ class Button(urwid.AttrWrap):
     """Styled button."""
     def __init__(self, *args, **kwargs):
         button = urwid.Button(*args, **kwargs)
-        padding = urwid.Padding(button, width=len(args[0]) + 4)
+        padding = urwid.Padding(button, width=wcswidth(args[0]) + 4)
         return super().__init__(padding, "button", "button_focused")
 
     def set_label(self, *args, **kwargs):
         self.original_widget.original_widget.set_label(*args, **kwargs)
-        self.original_widget.width = len(args[0]) + 4
+        self.original_widget.width = wcswidth(args[0]) + 4
 
 
 class CheckBox(urwid.AttrWrap):

From d65cac1215ecf16f12d8f8abb0f0d4bc68b719f5 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 08/55] Add basic support for Mastodon Lists

Fixes issue #255
---
 toot/tui/app.py      | 24 +++++++++++++++++++++---
 toot/tui/overlays.py | 16 +++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/toot/tui/app.py b/toot/tui/app.py
index 4ba25ab5..b49dee23 100644
--- a/toot/tui/app.py
+++ b/toot/tui/app.py
@@ -398,7 +398,9 @@ def _post(timeline, *args):
 
     def show_goto_menu(self):
         user_timelines = self.config.get("timelines", {})
-        menu = GotoMenu(user_timelines)
+        user_lists = api.get_lists(self.app, self.user) or []
+
+        menu = GotoMenu(user_timelines, user_lists)
         urwid.connect_signal(menu, "home_timeline",
             lambda x: self.goto_home_timeline())
         urwid.connect_signal(menu, "public_timeline",
@@ -411,10 +413,12 @@ def show_goto_menu(self):
             lambda x, local: self.goto_conversations())
         urwid.connect_signal(menu, "hashtag_timeline",
             lambda x, tag, local: self.goto_tag_timeline(tag, local=local))
+        urwid.connect_signal(menu, "list_timeline",
+            lambda x, list_item: self.goto_list_timeline(list_item))
 
         self.open_overlay(menu, title="Go to", options=dict(
             align="center", width=("relative", 60),
-            valign="middle", height=16 + len(user_timelines),
+            valign="middle", height=17 + len(user_timelines) + len(user_lists),
         ))
 
     def show_help(self):
@@ -468,6 +472,13 @@ def goto_tag_timeline(self, tag, local):
         )
         promise.add_done_callback(lambda *args: self.close_overlay())
 
+    def goto_list_timeline(self, list_item):
+        self.timeline_generator = api.timeline_list_generator(
+            self.app, self.user, list_item['id'], limit=40)
+        promise = self.async_load_timeline(
+            is_initial=True, timeline_name=f"\N{clipboard}{list_item['title']}")
+        promise.add_done_callback(lambda *args: self.close_overlay())
+
     def show_media(self, status):
         urls = [m["url"] for m in status.original.data["media_attachments"]]
         if urls:
@@ -660,12 +671,19 @@ def close_overlay(self):
 
     def refresh_timeline(self):
         # No point in refreshing the bookmarks timeline
-        if not self.timeline or self.timeline.name == 'bookmarks':
+        # and we don't have a good way to refresh a
+        # list timeline yet (no reference to list ID kept)
+        if (not self.timeline
+                or self.timeline.name == 'bookmarks'
+                or self.timeline.name.startswith("\N{clipboard}")):
             return
 
         if self.timeline.name.startswith("#"):
             self.timeline_generator = api.tag_timeline_generator(
                 self.app, self.user, self.timeline.name[1:], limit=40)
+        elif self.timeline.name.startswith("\N{clipboard}"):
+            self.timeline_generator = api.tag_timeline_generator(
+                self.app, self.user, self.timeline.name[1:], limit=40)
         else:
             if self.timeline.name.endswith("public"):
                 self.timeline_generator = api.public_timeline_generator(
diff --git a/toot/tui/overlays.py b/toot/tui/overlays.py
index 58b902aa..a9c24428 100644
--- a/toot/tui/overlays.py
+++ b/toot/tui/overlays.py
@@ -102,20 +102,21 @@ class GotoMenu(urwid.ListBox):
         "bookmark_timeline",
         "notification_timeline",
         "conversation_timeline",
+        "list_timeline",
     ]
 
-    def __init__(self, user_timelines):
+    def __init__(self, user_timelines, user_lists):
         self.hash_edit = EditBox(caption="Hashtag: ")
         self.message_widget = urwid.Text("")
 
-        actions = list(self.generate_actions(user_timelines))
+        actions = list(self.generate_actions(user_timelines, user_lists))
         walker = urwid.SimpleFocusListWalker(actions)
         super().__init__(walker)
 
     def get_hashtag(self):
         return self.hash_edit.edit_text.strip().lstrip("#")
 
-    def generate_actions(self, user_timelines):
+    def generate_actions(self, user_timelines, user_lists):
         def _home(button):
             self._emit("home_timeline")
 
@@ -147,6 +148,11 @@ def on_press(btn):
                 self._emit("hashtag_timeline", tag, local)
             return on_press
 
+        def mk_on_press_user_list(list_item):
+            def on_press(btn):
+                self._emit("list_timeline", list_item)
+            return on_press
+
         yield Button("Home timeline", on_press=_home)
         yield Button("Local public timeline", on_press=_local_public)
         yield Button("Global public timeline", on_press=_global_public)
@@ -164,6 +170,10 @@ def on_press(btn):
                 yield Button(f"#{tag}" + (" (local)" if is_local else ""),
                      on_press=mk_on_press_user_hashtag(tag, is_local))
 
+        for list_item in user_lists:
+            yield Button(f"\N{clipboard}{list_item['title']}",
+                         on_press=mk_on_press_user_list(list_item))
+
         yield urwid.Divider()
         yield self.hash_edit
         yield Button("Local hashtag timeline", on_press=lambda x: _hashtag(True))

From 7189c6b64f65aac8b987b29fd264325f227af3a1 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 09/55] "toot list" console command added

---
 toot/commands.py   |  8 +++++++-
 toot/console.py    |  9 +++++++++
 toot/output.py     | 11 +++++++++++
 toot/tui/scroll.py |  4 ++--
 4 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/toot/commands.py b/toot/commands.py
index e105a8dd..1e45dc42 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -1,3 +1,4 @@
+
 import sys
 import platform
 
@@ -8,7 +9,7 @@
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import (print_out, print_instance, print_account, print_acct_list,
                          print_search_results, print_timeline, print_notifications,
-                         print_tag_list)
+                         print_tag_list, print_list_list)
 from toot.tui.utils import parse_datetime
 from toot.utils import args_get_instance, delete_tmp_status_file, editor_input, multiline_input, EOF_KEY
 
@@ -423,6 +424,11 @@ def tags_followed(app, user, args):
     print_tag_list(response)
 
 
+def lists(app, user, args):
+    response = api.get_lists(app, user)
+    print_list_list(response)
+
+
 def mute(app, user, args):
     account = find_account(app, user, args.account)
     api.mute(app, user, account['id'])
diff --git a/toot/console.py b/toot/console.py
index 0a3acb50..bd261484 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -724,6 +724,14 @@ def editor(value):
     ),
 ]
 
+LIST_COMMANDS = [
+    Command(
+        name="lists",
+        description="List all user lists",
+        arguments=[],
+        require_auth=True,
+    ),
+]
 COMMAND_GROUPS = [
     ("Authentication", AUTH_COMMANDS),
     ("TUI", TUI_COMMANDS),
@@ -732,6 +740,7 @@ def editor(value):
     ("Status", STATUS_COMMANDS),
     ("Accounts", ACCOUNTS_COMMANDS),
     ("Hashtags", TAG_COMMANDS),
+    ("Lists", LIST_COMMANDS),
 ]
 
 COMMANDS = list(chain(*[commands for _, commands in COMMAND_GROUPS]))
diff --git a/toot/output.py b/toot/output.py
index 5be6a922..ba8c54e6 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -210,6 +210,17 @@ def print_tag_list(tags):
         print_out("You're not following any hashtags.")
 
 
+def print_list_list(lists):
+    if lists:
+        for list_item in lists:
+            replies_policy = list_item['replies_policy'] if list_item['replies_policy'] else ''
+            print_out(f"Name: <green>\"{list_item['title']}\"</green>\t"
+                      + f"ID: <green>{list_item['id']}\t</green>"
+                      + f"Replies policy: <green>{replies_policy}</green>")
+    else:
+        print_out("You have no lists defined.")
+
+
 def print_search_results(results):
     accounts = results['accounts']
     hashtags = results['hashtags']
diff --git a/toot/tui/scroll.py b/toot/tui/scroll.py
index fa2c3bb0..7626e84b 100644
--- a/toot/tui/scroll.py
+++ b/toot/tui/scroll.py
@@ -1,7 +1,7 @@
 # scroll.py
 #
 # Copied from the stig project by rndusr@github
-# https://github.com/rndusr/stig
+# https://github.com/rndusr/sti
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -423,4 +423,4 @@ def mouse_event(self, size, event, button, col, row, focus):
                 ow.set_scrollpos(pos + 1)
                 return True
 
-        return False
\ No newline at end of file
+        return False

From 94315dd65f580983acddfc4871c77ab995b43372 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 10/55] added "toot list_accounts" command

---
 toot/api.py      | 16 ++++++++++++++++
 toot/commands.py |  7 ++++++-
 toot/console.py  | 12 ++++++++++++
 toot/output.py   |  8 ++++++++
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/toot/api.py b/toot/api.py
index 7752af42..d0edb889 100644
--- a/toot/api.py
+++ b/toot/api.py
@@ -524,3 +524,19 @@ def get_instance(base_url):
 def get_lists(app, user):
     path = "/api/v1/lists"
     return _get_response_list(app, user, path)
+
+
+def find_list_id(app, user, title):
+    lists = get_lists(app, user)
+    for list_item in lists:
+        if list_item["title"] == title:
+            return list_item["id"]
+    return None
+
+
+def get_list_accounts(app, user, title):
+    id = find_list_id(app, user, title)
+    if id:
+        path = "/api/v1/{id}/accounts"
+        return _get_response_list(app, user, path)
+    return []
diff --git a/toot/commands.py b/toot/commands.py
index 1e45dc42..cc513ede 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -9,7 +9,7 @@
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import (print_out, print_instance, print_account, print_acct_list,
                          print_search_results, print_timeline, print_notifications,
-                         print_tag_list, print_list_list)
+                         print_tag_list, print_list_list, print_list_accounts)
 from toot.tui.utils import parse_datetime
 from toot.utils import args_get_instance, delete_tmp_status_file, editor_input, multiline_input, EOF_KEY
 
@@ -429,6 +429,11 @@ def lists(app, user, args):
     print_list_list(response)
 
 
+def list_accounts(app, user, args):
+    response = api.get_list_accounts(app, user, args.title)
+    print_list_accounts(args.title[0], response)
+
+
 def mute(app, user, args):
     account = find_account(app, user, args.account)
     api.mute(app, user, account['id'])
diff --git a/toot/console.py b/toot/console.py
index bd261484..146e5b67 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -731,6 +731,18 @@ def editor(value):
         arguments=[],
         require_auth=True,
     ),
+    Command(
+        name="list_accounts",
+        description="List the accounts in a list",
+        arguments=[
+            (["--title"], {
+                "action": "append",
+                "type": str,
+                "help": "title of the list"
+            }),
+        ],
+        require_auth=True,
+    ),
 ]
 COMMAND_GROUPS = [
     ("Authentication", AUTH_COMMANDS),
diff --git a/toot/output.py b/toot/output.py
index ba8c54e6..1114748e 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -221,6 +221,14 @@ def print_list_list(lists):
         print_out("You have no lists defined.")
 
 
+def print_list_accounts(list_title, accounts):
+    print_out(f"Accounts in list <green>\"{list_title}\"</green>:\n")
+    if accounts:
+        print_acct_list(accounts)
+    else:
+        print_out("This list has no accounts.")
+
+
 def print_search_results(results):
     accounts = results['accounts']
     hashtags = results['hashtags']

From 85c228bbfe04fee08b9c90f7452d387002d8e4d3 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:36 -0400
Subject: [PATCH 11/55] Added "toot list_delete" and "toot list_create"
 commands

---
 requirements.txt   |  1 +
 toot/api.py        | 21 +++++++++++++++------
 toot/commands.py   | 16 ++++++++++++++--
 toot/console.py    | 40 ++++++++++++++++++++++++++++++++++++++--
 toot/http.py       |  4 ++--
 toot/output.py     |  6 +++---
 toot/tui/scroll.py |  2 +-
 7 files changed, 74 insertions(+), 16 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 67ddf983..3616ac32 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ requests>=2.13,<3.0
 beautifulsoup4>=4.5.0,<5.0
 wcwidth>=0.1.7
 urwid>=2.0.0,<3.0
+
diff --git a/toot/api.py b/toot/api.py
index d0edb889..b4e0bd17 100644
--- a/toot/api.py
+++ b/toot/api.py
@@ -534,9 +534,18 @@ def find_list_id(app, user, title):
     return None
 
 
-def get_list_accounts(app, user, title):
-    id = find_list_id(app, user, title)
-    if id:
-        path = "/api/v1/{id}/accounts"
-        return _get_response_list(app, user, path)
-    return []
+def get_list_accounts(app, user, list_id):
+    path = f"/api/v1/lists/{list_id}/accounts"
+    return _get_response_list(app, user, path)
+
+
+def create_list(app, user, title, replies_policy):
+    url = "/api/v1/lists"
+    json = {'title': title}
+    if replies_policy:
+        json['replies_policy'] = replies_policy
+    return http.post(app, user, url, json=json).json()
+
+
+def delete_list(app, user, id):
+    return http.delete(app, user, f"/api/v1/lists/{id}")
diff --git a/toot/commands.py b/toot/commands.py
index cc513ede..5ac4aca2 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -430,8 +430,20 @@ def lists(app, user, args):
 
 
 def list_accounts(app, user, args):
-    response = api.get_list_accounts(app, user, args.title)
-    print_list_accounts(args.title[0], response)
+    id = args.id if args.id else api.find_list_id(app, user, args.title)
+    response = api.get_list_accounts(app, user, id)
+    print_list_accounts(response)
+
+
+def list_create(app, user, args):
+    api.create_list(app, user, title=args.title, replies_policy=args.replies_policy)
+    print_out(f"<green>✓ List \"{args.title}\" created.</green>")
+
+
+def list_delete(app, user, args):
+    id = args.id if args.id else api.find_list_id(app, user, args.title)
+    api.delete_list(app, user, id)
+    print_out(f"<green>✓ List \"{args.title}\"</green> <red>deleted.</red>")
 
 
 def mute(app, user, args):
diff --git a/toot/console.py b/toot/console.py
index 146e5b67..b53f3394 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -727,16 +727,52 @@ def editor(value):
 LIST_COMMANDS = [
     Command(
         name="lists",
-        description="List all user lists",
+        description="List all lists",
         arguments=[],
         require_auth=True,
     ),
     Command(
         name="list_accounts",
         description="List the accounts in a list",
+        arguments=[(["--id"], {
+            "type": str,
+            "help": "ID of the list"
+        }),
+            (["--title"], {
+                "type": str,
+                "help": "title of the list"
+            }),
+        ],
+        require_auth=True,
+    ),
+    Command(
+        name="list_create",
+        description="Create a list",
         arguments=[
+            (["--id"], {
+                "type": str,
+                "help": "ID of the list"
+            }),
+            (["--title"], {
+                "type": str,
+                "help": "title of the list"
+            }),
+            (["--replies-policy"], {
+                "type": str,
+                "help": "replies policy: 'followed', 'list', or 'none' (defaults to 'none')"
+            }),
+        ],
+        require_auth=True,
+    ),
+    Command(
+        name="list_delete",
+        description="Delete a list",
+        arguments=[
+            (["--id"], {
+                "type": str,
+                "help": "ID of the list"
+            }),
             (["--title"], {
-                "action": "append",
                 "type": str,
                 "help": "title of the list"
             }),
diff --git a/toot/http.py b/toot/http.py
index 597edc90..4e62bda8 100644
--- a/toot/http.py
+++ b/toot/http.py
@@ -92,13 +92,13 @@ def patch(app, user, path, headers=None, files=None, data=None, json=None):
     return process_response(response)
 
 
-def delete(app, user, path, data=None, headers=None):
+def delete(app, user, path, data=None, json=None, headers=None):
     url = app.base_url + path
 
     headers = headers or {}
     headers["Authorization"] = f"Bearer {user.access_token}"
 
-    request = Request('DELETE', url, headers=headers, json=data)
+    request = Request('DELETE', url, headers=headers, data=data, json=json)
     response = send_request(request)
 
     return process_response(response)
diff --git a/toot/output.py b/toot/output.py
index 1114748e..25f72529 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -214,15 +214,15 @@ def print_list_list(lists):
     if lists:
         for list_item in lists:
             replies_policy = list_item['replies_policy'] if list_item['replies_policy'] else ''
-            print_out(f"Name: <green>\"{list_item['title']}\"</green>\t"
+            print_out(f"Title: <green>\"{list_item['title']}\"</green>\t"
                       + f"ID: <green>{list_item['id']}\t</green>"
                       + f"Replies policy: <green>{replies_policy}</green>")
     else:
         print_out("You have no lists defined.")
 
 
-def print_list_accounts(list_title, accounts):
-    print_out(f"Accounts in list <green>\"{list_title}\"</green>:\n")
+def print_list_accounts(accounts):
+    print_out("Accounts in list</green>:\n")
     if accounts:
         print_acct_list(accounts)
     else:
diff --git a/toot/tui/scroll.py b/toot/tui/scroll.py
index 7626e84b..fe89be88 100644
--- a/toot/tui/scroll.py
+++ b/toot/tui/scroll.py
@@ -1,7 +1,7 @@
 # scroll.py
 #
 # Copied from the stig project by rndusr@github
-# https://github.com/rndusr/sti
+# https://github.com/rndusr/stig
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

From 678cc19ccfe2528353d3843395cc6dab7a873a19 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 12/55] Added toot list_add_account command

---
 toot/api.py      | 12 ++++++++++++
 toot/commands.py | 16 ++++++++++++++++
 toot/console.py  | 19 +++++++++++++++++++
 3 files changed, 47 insertions(+)

diff --git a/toot/api.py b/toot/api.py
index b4e0bd17..3d9e9569 100644
--- a/toot/api.py
+++ b/toot/api.py
@@ -549,3 +549,15 @@ def create_list(app, user, title, replies_policy):
 
 def delete_list(app, user, id):
     return http.delete(app, user, f"/api/v1/lists/{id}")
+
+
+def add_accounts_to_list(app, user, list_id, account_ids):
+    url = f"/api/v1/lists/{list_id}/accounts"
+    json = {'account_ids': account_ids}
+    return http.post(app, user, url, json=json).json()
+
+
+def remove_accounts_from_list(app, user, list_id, account_ids):
+    url = f"/api/v1/lists/{list_id}/accounts"
+    json = {'account_ids[]': account_ids}
+    return http.delete(app, user, url, json=json)
diff --git a/toot/commands.py b/toot/commands.py
index 5ac4aca2..4a7c9082 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -446,6 +446,22 @@ def list_delete(app, user, args):
     print_out(f"<green>✓ List \"{args.title}\"</green> <red>deleted.</red>")
 
 
+def list_add_account(app, user, args):
+    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
+    if not list_id:
+        print_out("<red>List not found</red>")
+        return
+    account = find_account(app, user, args.account)
+    if not account:
+        print_out("<red>Account not found</red>")
+        return
+    try:
+        api.add_accounts_to_list(app, user, list_id, [account['id']])
+        print_out(f"<green>✓ Added account \"{account['acct']}\"</green>")
+    except Exception as ex:
+        print_out(f"<red>{ex}</red>")
+
+
 def mute(app, user, args):
     account = find_account(app, user, args.account)
     api.mute(app, user, account['id'])
diff --git a/toot/console.py b/toot/console.py
index b53f3394..cfe20ff8 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -779,6 +779,25 @@ def editor(value):
         ],
         require_auth=True,
     ),
+    Command(
+        name="list_add_account",
+        description="Add account to list",
+        arguments=[
+            (["--id"], {
+                "type": str,
+                "help": "ID of the list"
+            }),
+            (["--title"], {
+                "type": str,
+                "help": "title of the list"
+            }),
+            (["--account"], {
+                "type": str,
+                "help": "Account to add"
+            }),
+        ],
+        require_auth=True,
+    ),
 ]
 COMMAND_GROUPS = [
     ("Authentication", AUTH_COMMANDS),

From 1164d0c5abe5c389e81e68c663f6367bce369e7d Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 13/55] Added toot list_remove_account command

---
 toot/api.py      |  2 +-
 toot/commands.py | 16 ++++++++++++++++
 toot/console.py  | 19 +++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/toot/api.py b/toot/api.py
index 3d9e9569..8557c741 100644
--- a/toot/api.py
+++ b/toot/api.py
@@ -559,5 +559,5 @@ def add_accounts_to_list(app, user, list_id, account_ids):
 
 def remove_accounts_from_list(app, user, list_id, account_ids):
     url = f"/api/v1/lists/{list_id}/accounts"
-    json = {'account_ids[]': account_ids}
+    json = {'account_ids': account_ids}
     return http.delete(app, user, url, json=json)
diff --git a/toot/commands.py b/toot/commands.py
index 4a7c9082..7120f2fe 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -462,6 +462,22 @@ def list_add_account(app, user, args):
         print_out(f"<red>{ex}</red>")
 
 
+def list_remove_account(app, user, args):
+    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
+    if not list_id:
+        print_out("<red>List not found</red>")
+        return
+    account = find_account(app, user, args.account)
+    if not account:
+        print_out("<red>Account not found</red>")
+        return
+    try:
+        api.remove_accounts_from_list(app, user, list_id, [account['id']])
+        print_out(f"<green>✓ Removed account \"{args.account}\"</green>")
+    except Exception as ex:
+        print_out(f"<red>{ex}</red>")
+
+
 def mute(app, user, args):
     account = find_account(app, user, args.account)
     api.mute(app, user, account['id'])
diff --git a/toot/console.py b/toot/console.py
index cfe20ff8..5583fe0b 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -798,6 +798,25 @@ def editor(value):
         ],
         require_auth=True,
     ),
+    Command(
+        name="list_remove_account",
+        description="Remove account from list",
+        arguments=[
+            (["--id"], {
+                "type": str,
+                "help": "ID of the list"
+            }),
+            (["--title"], {
+                "type": str,
+                "help": "title of the list"
+            }),
+            (["--account"], {
+                "type": str,
+                "help": "Account to remove"
+            }),
+        ],
+        require_auth=True,
+    ),
 ]
 COMMAND_GROUPS = [
     ("Authentication", AUTH_COMMANDS),

From 7ad9e7cd8149f15f0128e7701e981254d493afff Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 14/55] minor improvement of feedback messages

---
 toot/commands.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/toot/commands.py b/toot/commands.py
index 7120f2fe..7ea29d9b 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -443,7 +443,7 @@ def list_create(app, user, args):
 def list_delete(app, user, args):
     id = args.id if args.id else api.find_list_id(app, user, args.title)
     api.delete_list(app, user, id)
-    print_out(f"<green>✓ List \"{args.title}\"</green> <red>deleted.</red>")
+    print_out(f"<green>✓ List \"{args.title if args.title else args.id}\"</green> <red>deleted.</red>")
 
 
 def list_add_account(app, user, args):
@@ -457,7 +457,7 @@ def list_add_account(app, user, args):
         return
     try:
         api.add_accounts_to_list(app, user, list_id, [account['id']])
-        print_out(f"<green>✓ Added account \"{account['acct']}\"</green>")
+        print_out(f"<green>✓ Added account \"{args.account}\"</green>")
     except Exception as ex:
         print_out(f"<red>{ex}</red>")
 

From 0430eed4d46b8b19a4b06089119b975dffa468f4 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 15/55] Changed parameters for list cmds

Title is now a positional parameter.

Also added some error handling in the command processing
for looking up list IDs per @ihabunek 's suggestions
---
 toot/commands.py | 34 ++++++++++++++++++----------------
 toot/console.py  | 35 ++++++++++++++++++-----------------
 2 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/toot/commands.py b/toot/commands.py
index 7ea29d9b..b7a74c73 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -430,8 +430,12 @@ def lists(app, user, args):
 
 
 def list_accounts(app, user, args):
-    id = args.id if args.id else api.find_list_id(app, user, args.title)
-    response = api.get_list_accounts(app, user, id)
+    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
+    if not list_id:
+        print_out("<red>List not found</red>")
+        return
+
+    response = api.get_list_accounts(app, user, list_id)
     print_list_accounts(response)
 
 
@@ -441,12 +445,16 @@ def list_create(app, user, args):
 
 
 def list_delete(app, user, args):
-    id = args.id if args.id else api.find_list_id(app, user, args.title)
-    api.delete_list(app, user, id)
+    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
+    if not list_id:
+        print_out("<red>List not found</red>")
+        return
+
+    api.delete_list(app, user, list_id)
     print_out(f"<green>✓ List \"{args.title if args.title else args.id}\"</green> <red>deleted.</red>")
 
 
-def list_add_account(app, user, args):
+def list_add(app, user, args):
     list_id = args.id if args.id else api.find_list_id(app, user, args.title)
     if not list_id:
         print_out("<red>List not found</red>")
@@ -455,14 +463,11 @@ def list_add_account(app, user, args):
     if not account:
         print_out("<red>Account not found</red>")
         return
-    try:
-        api.add_accounts_to_list(app, user, list_id, [account['id']])
-        print_out(f"<green>✓ Added account \"{args.account}\"</green>")
-    except Exception as ex:
-        print_out(f"<red>{ex}</red>")
+    api.add_accounts_to_list(app, user, list_id, [account['id']])
+    print_out(f"<green>✓ Added account \"{args.account}\"</green>")
 
 
-def list_remove_account(app, user, args):
+def list_remove(app, user, args):
     list_id = args.id if args.id else api.find_list_id(app, user, args.title)
     if not list_id:
         print_out("<red>List not found</red>")
@@ -471,11 +476,8 @@ def list_remove_account(app, user, args):
     if not account:
         print_out("<red>Account not found</red>")
         return
-    try:
-        api.remove_accounts_from_list(app, user, list_id, [account['id']])
-        print_out(f"<green>✓ Removed account \"{args.account}\"</green>")
-    except Exception as ex:
-        print_out(f"<red>{ex}</red>")
+    api.remove_accounts_from_list(app, user, list_id, [account['id']])
+    print_out(f"<green>✓ Removed account \"{args.account}\"</green>")
 
 
 def mute(app, user, args):
diff --git a/toot/console.py b/toot/console.py
index 5583fe0b..5c84d97c 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -734,12 +734,14 @@ def editor(value):
     Command(
         name="list_accounts",
         description="List the accounts in a list",
-        arguments=[(["--id"], {
-            "type": str,
-            "help": "ID of the list"
-        }),
-            (["--title"], {
+        arguments=[
+            (["--id"], {
+                "type": str,
+                "help": "ID of the list"
+            }),
+            (["title"], {
                 "type": str,
+                "nargs": "?",
                 "help": "title of the list"
             }),
         ],
@@ -749,11 +751,7 @@ def editor(value):
         name="list_create",
         description="Create a list",
         arguments=[
-            (["--id"], {
-                "type": str,
-                "help": "ID of the list"
-            }),
-            (["--title"], {
+            (["title"], {
                 "type": str,
                 "help": "title of the list"
             }),
@@ -772,26 +770,28 @@ def editor(value):
                 "type": str,
                 "help": "ID of the list"
             }),
-            (["--title"], {
+            (["title"], {
                 "type": str,
+                "nargs": "?",
                 "help": "title of the list"
             }),
         ],
         require_auth=True,
     ),
     Command(
-        name="list_add_account",
+        name="list_add",
         description="Add account to list",
         arguments=[
             (["--id"], {
                 "type": str,
                 "help": "ID of the list"
             }),
-            (["--title"], {
+            (["title"], {
                 "type": str,
+                "nargs": "?",
                 "help": "title of the list"
             }),
-            (["--account"], {
+            (["account"], {
                 "type": str,
                 "help": "Account to add"
             }),
@@ -799,18 +799,19 @@ def editor(value):
         require_auth=True,
     ),
     Command(
-        name="list_remove_account",
+        name="list_remove",
         description="Remove account from list",
         arguments=[
             (["--id"], {
                 "type": str,
                 "help": "ID of the list"
             }),
-            (["--title"], {
+            (["title"], {
                 "type": str,
+                "nargs": "?",
                 "help": "title of the list"
             }),
-            (["--account"], {
+            (["account"], {
                 "type": str,
                 "help": "Account to remove"
             }),

From 45cffb6ac04682fce9a20ba09723701f0a185551 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 16/55] Give a more specfic error message if we can't add an
 account to list

---
 toot/commands.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/toot/commands.py b/toot/commands.py
index b7a74c73..a2879e94 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -463,7 +463,23 @@ def list_add(app, user, args):
     if not account:
         print_out("<red>Account not found</red>")
         return
-    api.add_accounts_to_list(app, user, list_id, [account['id']])
+    try:
+        api.add_accounts_to_list(app, user, list_id, [account['id']])
+    except Exception as ex:
+        # if we failed to add the account, try to give a
+        # more specific error message than "record not found"
+        my_accounts = api.followers(app, user, account['id'])
+        found = False
+        if my_accounts:
+            for my_account in my_accounts:
+                if my_account['id'] == account['id']:
+                    found = True
+                    break
+        if found is False:
+            print_out(f"<red>You must follow @{account['acct']} before adding this account to a list.</red>")
+        else:
+            print_out(f"<red>{ex}</red>")
+        return
     print_out(f"<green>✓ Added account \"{args.account}\"</green>")
 
 

From 037f7f97049fdfb9fd85c3d2e97100c0adc5b31b Mon Sep 17 00:00:00 2001
From: Ivan Habunek <ivan@habunek.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 17/55] Break up integration tests

---
 tests/integration/__init__.py      |   0
 tests/integration/conftest.py      | 122 +++++
 tests/integration/test_accounts.py |  19 +
 tests/integration/test_auth.py     | 125 +++++
 tests/integration/test_post.py     | 288 +++++++++++
 tests/integration/test_read.py     |  83 ++++
 tests/integration/test_status.py   |  89 ++++
 tests/test_integration.py          | 758 -----------------------------
 8 files changed, 726 insertions(+), 758 deletions(-)
 create mode 100644 tests/integration/__init__.py
 create mode 100644 tests/integration/conftest.py
 create mode 100644 tests/integration/test_accounts.py
 create mode 100644 tests/integration/test_auth.py
 create mode 100644 tests/integration/test_post.py
 create mode 100644 tests/integration/test_read.py
 create mode 100644 tests/integration/test_status.py
 delete mode 100644 tests/test_integration.py

diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
new file mode 100644
index 00000000..b4aaa1ef
--- /dev/null
+++ b/tests/integration/conftest.py
@@ -0,0 +1,122 @@
+"""
+This module contains integration tests meant to run against a test Mastodon instance.
+
+You can set up a test instance locally by following this guide:
+https://docs.joinmastodon.org/dev/setup/
+
+To enable integration tests, export the following environment variables to match
+your test server and database:
+
+```
+export TOOT_TEST_HOSTNAME="localhost:3000"
+export TOOT_TEST_DATABASE_DSN="dbname=mastodon_development"
+```
+"""
+
+import re
+import os
+import psycopg2
+import pytest
+import uuid
+
+from pathlib import Path
+from toot import api, App, User
+from toot.console import run_command
+
+# Host name of a test instance to run integration tests against
+# DO NOT USE PUBLIC INSTANCES!!!
+BASE_URL = os.getenv("TOOT_TEST_BASE_URL")
+
+# Mastodon database name, used to confirm user registration without having to click the link
+DATABASE_DSN = os.getenv("TOOT_TEST_DATABASE_DSN")
+
+# Toot logo used for testing image upload
+TRUMPET = str(Path(__file__).parent.parent.parent / "trumpet.png")
+
+ASSETS_DIR = str(Path(__file__).parent.parent / "assets")
+
+
+if not BASE_URL or not DATABASE_DSN:
+    pytest.skip("Skipping integration tests", allow_module_level=True)
+
+# ------------------------------------------------------------------------------
+# Fixtures
+# ------------------------------------------------------------------------------
+
+
+def create_app():
+    instance = api.get_instance(BASE_URL)
+    response = api.create_app(BASE_URL)
+    return App(instance["uri"], BASE_URL, response["client_id"], response["client_secret"])
+
+
+def register_account(app: App):
+    username = str(uuid.uuid4())[-10:]
+    email = f"{username}@example.com"
+
+    response = api.register_account(app, username, email, "password", "en")
+    confirm_user(email)
+    return User(app.instance, username, response["access_token"])
+
+
+def confirm_user(email):
+    conn = psycopg2.connect(DATABASE_DSN)
+    cursor = conn.cursor()
+    cursor.execute("UPDATE users SET confirmed_at = now() WHERE email = %s;", (email,))
+    conn.commit()
+
+
+@pytest.fixture(scope="session")
+def app():
+    return create_app()
+
+
+@pytest.fixture(scope="session")
+def user(app):
+    return register_account(app)
+
+
+@pytest.fixture(scope="session")
+def friend(app):
+    return register_account(app)
+
+
+@pytest.fixture
+def run(app, user, capsys):
+    def _run(command, *params, as_user=None):
+        run_command(app, as_user or user, command, params or [])
+        out, err = capsys.readouterr()
+        assert err == ""
+        return strip_ansi(out)
+    return _run
+
+
+@pytest.fixture
+def run_anon(capsys):
+    def _run(command, *params):
+        run_command(None, None, command, params or [])
+        out, err = capsys.readouterr()
+        assert err == ""
+        return strip_ansi(out)
+    return _run
+
+
+# ------------------------------------------------------------------------------
+# Utils
+# ------------------------------------------------------------------------------
+
+strip_ansi_pattern = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+
+
+def strip_ansi(string):
+    return strip_ansi_pattern.sub("", string).strip()
+
+
+def posted_status_id(out):
+    pattern = re.compile(r"Toot posted: http://([^/]+)/([^/]+)/(.+)")
+    match = re.search(pattern, out)
+    assert match
+
+    _, _, status_id = match.groups()
+
+    return status_id
diff --git a/tests/integration/test_accounts.py b/tests/integration/test_accounts.py
new file mode 100644
index 00000000..bf01f1bc
--- /dev/null
+++ b/tests/integration/test_accounts.py
@@ -0,0 +1,19 @@
+
+
+def test_whoami(user, run):
+    out = run("whoami")
+    # TODO: test other fields once updating account is supported
+    assert f"@{user.username}" in out
+
+
+def test_whois(app, friend, run):
+    variants = [
+        friend.username,
+        f"@{friend.username}",
+        f"{friend.username}@{app.instance}",
+        f"@{friend.username}@{app.instance}",
+    ]
+
+    for username in variants:
+        out = run("whois", username)
+        assert f"@{friend.username}" in out
diff --git a/tests/integration/test_auth.py b/tests/integration/test_auth.py
new file mode 100644
index 00000000..afe5c39d
--- /dev/null
+++ b/tests/integration/test_auth.py
@@ -0,0 +1,125 @@
+import pytest
+
+from tests.integration.conftest import TRUMPET
+from toot import api
+from toot.exceptions import ConsoleError
+from toot.utils import get_text
+
+
+def test_update_account_no_options(run):
+    with pytest.raises(ConsoleError) as exc:
+        run("update_account")
+    assert str(exc.value) == "Please specify at least one option to update the account"
+
+
+def test_update_account_display_name(run, app, user):
+    out = run("update_account", "--display-name", "elwood")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["display_name"] == "elwood"
+
+
+def test_update_account_note(run, app, user):
+    note = ("It's 106 miles to Chicago, we got a full tank of gas, half a pack "
+           "of cigarettes, it's dark... and we're wearing sunglasses.")
+
+    out = run("update_account", "--note", note)
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert get_text(account["note"]) == note
+
+
+def test_update_account_language(run, app, user):
+    out = run("update_account", "--language", "hr")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["source"]["language"] == "hr"
+
+
+def test_update_account_privacy(run, app, user):
+    out = run("update_account", "--privacy", "private")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["source"]["privacy"] == "private"
+
+
+def test_update_account_avatar(run, app, user):
+    account = api.verify_credentials(app, user)
+    old_value = account["avatar"]
+
+    out = run("update_account", "--avatar", TRUMPET)
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["avatar"] != old_value
+
+
+def test_update_account_header(run, app, user):
+    account = api.verify_credentials(app, user)
+    old_value = account["header"]
+
+    out = run("update_account", "--header", TRUMPET)
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["header"] != old_value
+
+
+def test_update_account_locked(run, app, user):
+    out = run("update_account", "--locked")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["locked"] is True
+
+    out = run("update_account", "--no-locked")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["locked"] is False
+
+
+def test_update_account_bot(run, app, user):
+    out = run("update_account", "--bot")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["bot"] is True
+
+    out = run("update_account", "--no-bot")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["bot"] is False
+
+
+def test_update_account_discoverable(run, app, user):
+    out = run("update_account", "--discoverable")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["discoverable"] is True
+
+    out = run("update_account", "--no-discoverable")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["discoverable"] is False
+
+
+def test_update_account_sensitive(run, app, user):
+    out = run("update_account", "--sensitive")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["source"]["sensitive"] is True
+
+    out = run("update_account", "--no-sensitive")
+    assert out == "✓ Account updated"
+
+    account = api.verify_credentials(app, user)
+    assert account["source"]["sensitive"] is False
diff --git a/tests/integration/test_post.py b/tests/integration/test_post.py
new file mode 100644
index 00000000..7ad7eb92
--- /dev/null
+++ b/tests/integration/test_post.py
@@ -0,0 +1,288 @@
+import re
+import uuid
+
+from datetime import datetime, timedelta, timezone
+from os import path
+from tests.integration.conftest import ASSETS_DIR, posted_status_id
+from toot import CLIENT_NAME, CLIENT_WEBSITE, api
+from toot.utils import get_text
+from unittest import mock
+
+
+def test_post(app, user, run):
+    text = "i wish i was a #lumberjack"
+    out = run("post", text)
+    status_id = posted_status_id(out)
+
+    status = api.fetch_status(app, user, status_id)
+    assert text == get_text(status["content"])
+    assert status["visibility"] == "public"
+    assert status["sensitive"] is False
+    assert status["spoiler_text"] == ""
+    assert status["poll"] is None
+
+    # Pleroma doesn't return the application
+    if status["application"]:
+        assert status["application"]["name"] == CLIENT_NAME
+        assert status["application"]["website"] == CLIENT_WEBSITE
+
+
+def test_post_visibility(app, user, run):
+    for visibility in ["public", "unlisted", "private", "direct"]:
+        out = run("post", "foo", "--visibility", visibility)
+        status_id = posted_status_id(out)
+        status = api.fetch_status(app, user, status_id)
+        assert status["visibility"] == visibility
+
+
+def test_post_scheduled_at(app, user, run):
+    text = str(uuid.uuid4())
+    scheduled_at = datetime.now(timezone.utc).replace(microsecond=0) + timedelta(minutes=10)
+
+    out = run("post", text, "--scheduled-at", scheduled_at.isoformat())
+    assert "Toot scheduled for" in out
+
+    statuses = api.scheduled_statuses(app, user)
+    [status] = [s for s in statuses if s["params"]["text"] == text]
+    assert datetime.strptime(status["scheduled_at"], "%Y-%m-%dT%H:%M:%S.%f%z") == scheduled_at
+
+
+def test_post_scheduled_in(app, user, run):
+    text = str(uuid.uuid4())
+
+    variants = [
+        ("1 day", timedelta(days=1)),
+        ("1 day 6 hours", timedelta(days=1, hours=6)),
+        ("1 day 6 hours 13 minutes", timedelta(days=1, hours=6, minutes=13)),
+        ("1 day 6 hours 13 minutes 51 second", timedelta(days=1, hours=6, minutes=13, seconds=51)),
+        ("2d", timedelta(days=2)),
+        ("2d6h", timedelta(days=2, hours=6)),
+        ("2d6h13m", timedelta(days=2, hours=6, minutes=13)),
+        ("2d6h13m51s", timedelta(days=2, hours=6, minutes=13, seconds=51)),
+    ]
+
+    datetimes = []
+    for scheduled_in, delta in variants:
+        out = run("post", text, "--scheduled-in", scheduled_in)
+        dttm = datetime.utcnow() + delta
+        assert out.startswith(f"Toot scheduled for: {str(dttm)[:16]}")
+        datetimes.append(dttm)
+
+    scheduled = api.scheduled_statuses(app, user)
+    scheduled = [s for s in scheduled if s["params"]["text"] == text]
+    scheduled = sorted(scheduled, key=lambda s: s["scheduled_at"])
+    assert len(scheduled) == 8
+
+    for expected, status in zip(datetimes, scheduled):
+        actual = datetime.strptime(status["scheduled_at"], "%Y-%m-%dT%H:%M:%S.%fZ")
+        delta = expected - actual
+        assert delta.total_seconds() < 5
+
+
+def test_post_poll(app, user, run):
+    text = str(uuid.uuid4())
+
+    out = run(
+        "post", text,
+        "--poll-option", "foo",
+        "--poll-option", "bar",
+        "--poll-option", "baz",
+        "--poll-option", "qux",
+    )
+
+    status_id = posted_status_id(out)
+
+    status = api.fetch_status(app, user, status_id)
+    assert status["poll"]["expired"] is False
+    assert status["poll"]["multiple"] is False
+    assert status["poll"]["options"] == [
+        {"title": "foo", "votes_count": 0},
+        {"title": "bar", "votes_count": 0},
+        {"title": "baz", "votes_count": 0},
+        {"title": "qux", "votes_count": 0}
+    ]
+
+    # Test expires_at is 24h by default
+    actual = datetime.strptime(status["poll"]["expires_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
+    expected = datetime.now(timezone.utc) + timedelta(days=1)
+    delta = actual - expected
+    assert delta.total_seconds() < 5
+
+
+def test_post_poll_multiple(app, user, run):
+    text = str(uuid.uuid4())
+
+    out = run(
+        "post", text,
+        "--poll-option", "foo",
+        "--poll-option", "bar",
+        "--poll-multiple"
+    )
+
+    status_id = posted_status_id(out)
+
+    status = api.fetch_status(app, user, status_id)
+    assert status["poll"]["multiple"] is True
+
+
+def test_post_poll_expires_in(app, user, run):
+    text = str(uuid.uuid4())
+
+    out = run(
+        "post", text,
+        "--poll-option", "foo",
+        "--poll-option", "bar",
+        "--poll-expires-in", "8h",
+    )
+
+    status_id = posted_status_id(out)
+
+    status = api.fetch_status(app, user, status_id)
+    actual = datetime.strptime(status["poll"]["expires_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
+    expected = datetime.now(timezone.utc) + timedelta(hours=8)
+    delta = actual - expected
+    assert delta.total_seconds() < 5
+
+
+def test_post_poll_hide_totals(app, user, run):
+    text = str(uuid.uuid4())
+
+    out = run(
+        "post", text,
+        "--poll-option", "foo",
+        "--poll-option", "bar",
+        "--poll-hide-totals"
+    )
+
+    status_id = posted_status_id(out)
+
+    status = api.fetch_status(app, user, status_id)
+
+    # votes_count is None when totals are hidden
+    assert status["poll"]["options"] == [
+        {"title": "foo", "votes_count": None},
+        {"title": "bar", "votes_count": None},
+    ]
+
+
+def test_post_language(app, user, run):
+    out = run("post", "test", "--language", "hr")
+    status_id = posted_status_id(out)
+    status = api.fetch_status(app, user, status_id)
+    assert status["language"] == "hr"
+
+    out = run("post", "test", "--language", "zh")
+    status_id = posted_status_id(out)
+    status = api.fetch_status(app, user, status_id)
+    assert status["language"] == "zh"
+
+
+def test_media_thumbnail(app, user, run):
+    video_path = path.join(ASSETS_DIR, "small.webm")
+    thumbnail_path = path.join(ASSETS_DIR, "test1.png")
+
+    out = run(
+        "post",
+        "--media", video_path,
+        "--thumbnail", thumbnail_path,
+        "--description", "foo",
+        "some text"
+    )
+
+    status_id = posted_status_id(out)
+    status = api.fetch_status(app, user, status_id)
+    [media] = status["media_attachments"]
+
+    assert media["description"] == "foo"
+    assert media["type"] == "video"
+    assert media["url"].endswith(".mp4")
+    assert media["preview_url"].endswith(".png")
+
+    # Video properties
+    assert int(media["meta"]["original"]["duration"]) == 5
+    assert media["meta"]["original"]["height"] == 320
+    assert media["meta"]["original"]["width"] == 560
+
+    # Thumbnail properties
+    assert media["meta"]["small"]["height"] == 50
+    assert media["meta"]["small"]["width"] == 50
+
+
+def test_media_attachments(app, user, run):
+    path1 = path.join(ASSETS_DIR, "test1.png")
+    path2 = path.join(ASSETS_DIR, "test2.png")
+    path3 = path.join(ASSETS_DIR, "test3.png")
+    path4 = path.join(ASSETS_DIR, "test4.png")
+
+    out = run(
+        "post",
+        "--media", path1,
+        "--media", path2,
+        "--media", path3,
+        "--media", path4,
+        "--description", "Test 1",
+        "--description", "Test 2",
+        "--description", "Test 3",
+        "--description", "Test 4",
+        "some text"
+    )
+
+    status_id = posted_status_id(out)
+    status = api.fetch_status(app, user, status_id)
+
+    [a1, a2, a3, a4] = status["media_attachments"]
+
+    # Pleroma doesn't send metadata
+    if "meta" in a1:
+        assert a1["meta"]["original"]["size"] == "50x50"
+        assert a2["meta"]["original"]["size"] == "50x60"
+        assert a3["meta"]["original"]["size"] == "50x70"
+        assert a4["meta"]["original"]["size"] == "50x80"
+
+    assert a1["description"] == "Test 1"
+    assert a2["description"] == "Test 2"
+    assert a3["description"] == "Test 3"
+    assert a4["description"] == "Test 4"
+
+
+@mock.patch("toot.utils.multiline_input")
+@mock.patch("sys.stdin.read")
+def test_media_attachment_without_text(mock_read, mock_ml, app, user, run):
+    # No status from stdin or readline
+    mock_read.return_value = ""
+    mock_ml.return_value = ""
+
+    media_path = path.join(ASSETS_DIR, "test1.png")
+
+    out = run("post", "--media", media_path)
+    status_id = posted_status_id(out)
+
+    status = api.fetch_status(app, user, status_id)
+    assert status["content"] == ""
+
+    [attachment] = status["media_attachments"]
+    assert not attachment["description"]
+
+    # Pleroma doesn't send metadata
+    if "meta" in attachment:
+        assert attachment["meta"]["original"]["size"] == "50x50"
+
+
+def test_reply_thread(app, user, friend, run):
+    status = api.post_status(app, friend, "This is the status")
+
+    out = run("post", "--reply-to", status["id"], "This is the reply")
+    status_id = posted_status_id(out)
+    reply = api.fetch_status(app, user, status_id)
+
+    assert reply["in_reply_to_id"] == status["id"]
+
+    out = run("thread", status["id"])
+    [s1, s2] = [s.strip() for s in re.split(r"─+", out) if s.strip()]
+
+    assert "This is the status" in s1
+    assert "This is the reply" in s2
+    assert friend.username in s1
+    assert user.username in s2
+    assert status["id"] in s1
+    assert reply["id"] in s2
diff --git a/tests/integration/test_read.py b/tests/integration/test_read.py
new file mode 100644
index 00000000..13f05d1f
--- /dev/null
+++ b/tests/integration/test_read.py
@@ -0,0 +1,83 @@
+import pytest
+
+from tests.integration.conftest import BASE_URL
+from toot import api
+from toot.exceptions import ConsoleError
+
+
+def test_instance(app, run):
+    out = run("instance", "--disable-https")
+    assert "Mastodon" in out
+    assert app.instance in out
+    assert "running Mastodon" in out
+
+
+def test_instance_anon(app, run_anon):
+    out = run_anon("instance", BASE_URL)
+    assert "Mastodon" in out
+    assert app.instance in out
+    assert "running Mastodon" in out
+
+    # Need to specify the instance name when running anon
+    with pytest.raises(ConsoleError) as exc:
+        run_anon("instance")
+    assert str(exc.value) == "Please specify an instance."
+
+
+def test_whoami(user, run):
+    out = run("whoami")
+    # TODO: test other fields once updating account is supported
+    assert f"@{user.username}" in out
+
+
+def test_whois(app, friend, run):
+    variants = [
+        friend.username,
+        f"@{friend.username}",
+        f"{friend.username}@{app.instance}",
+        f"@{friend.username}@{app.instance}",
+    ]
+
+    for username in variants:
+        out = run("whois", username)
+        assert f"@{friend.username}" in out
+
+
+def test_search_account(friend, run):
+    out = run("search", friend.username)
+    assert out == f"Accounts:\n* @{friend.username}"
+
+
+def test_search_hashtag(app, user, run):
+    api.post_status(app, user, "#hashtag_x")
+    api.post_status(app, user, "#hashtag_y")
+    api.post_status(app, user, "#hashtag_z")
+
+    out = run("search", "#hashtag")
+    assert out == "Hashtags:\n#hashtag_x, #hashtag_y, #hashtag_z"
+
+
+def test_tags(run):
+    out = run("tags_followed")
+    assert out == "You're not following any hashtags."
+
+    out = run("tags_follow", "foo")
+    assert out == "✓ You are now following #foo"
+
+    out = run("tags_followed")
+    assert out == f"* #foo\t{BASE_URL}/tags/foo"
+
+    out = run("tags_follow", "bar")
+    assert out == "✓ You are now following #bar"
+
+    out = run("tags_followed")
+    assert out == "\n".join([
+        f"* #bar\t{BASE_URL}/tags/bar",
+        f"* #foo\t{BASE_URL}/tags/foo",
+    ])
+
+    out = run("tags_unfollow", "foo")
+    assert out == "✓ You are no longer following #foo"
+
+    out = run("tags_followed")
+    assert out == f"* #bar\t{BASE_URL}/tags/bar"
diff --git a/tests/integration/test_status.py b/tests/integration/test_status.py
new file mode 100644
index 00000000..82741740
--- /dev/null
+++ b/tests/integration/test_status.py
@@ -0,0 +1,89 @@
+import time
+import pytest
+
+from toot import api
+from toot.exceptions import NotFoundError
+
+
+def test_delete_status(app, user, run):
+    status = api.post_status(app, user, "foo")
+
+    out = run("delete", status["id"])
+    assert out == "✓ Status deleted"
+
+    with pytest.raises(NotFoundError):
+        api.fetch_status(app, user, status["id"])
+
+
+def test_favourite(app, user, run):
+    status = api.post_status(app, user, "foo")
+    assert not status["favourited"]
+
+    out = run("favourite", status["id"])
+    assert out == "✓ Status favourited"
+
+    status = api.fetch_status(app, user, status["id"])
+    assert status["favourited"]
+
+    out = run("unfavourite", status["id"])
+    assert out == "✓ Status unfavourited"
+
+    # A short delay is required before the server returns new data
+    time.sleep(0.1)
+
+    status = api.fetch_status(app, user, status["id"])
+    assert not status["favourited"]
+
+
+def test_reblog(app, user, run):
+    status = api.post_status(app, user, "foo")
+    assert not status["reblogged"]
+
+    out = run("reblog", status["id"])
+    assert out == "✓ Status reblogged"
+
+    status = api.fetch_status(app, user, status["id"])
+    assert status["reblogged"]
+
+    out = run("reblogged_by", status["id"])
+    assert out == f"@{user.username}"
+
+    out = run("unreblog", status["id"])
+    assert out == "✓ Status unreblogged"
+
+    status = api.fetch_status(app, user, status["id"])
+    assert not status["reblogged"]
+
+
+def test_pin(app, user, run):
+    status = api.post_status(app, user, "foo")
+    assert not status["pinned"]
+
+    out = run("pin", status["id"])
+    assert out == "✓ Status pinned"
+
+    status = api.fetch_status(app, user, status["id"])
+    assert status["pinned"]
+
+    out = run("unpin", status["id"])
+    assert out == "✓ Status unpinned"
+
+    status = api.fetch_status(app, user, status["id"])
+    assert not status["pinned"]
+
+
+def test_bookmark(app, user, run):
+    status = api.post_status(app, user, "foo")
+    assert not status["bookmarked"]
+
+    out = run("bookmark", status["id"])
+    assert out == "✓ Status bookmarked"
+
+    status = api.fetch_status(app, user, status["id"])
+    assert status["bookmarked"]
+
+    out = run("unbookmark", status["id"])
+    assert out == "✓ Status unbookmarked"
+
+    status = api.fetch_status(app, user, status["id"])
+    assert not status["bookmarked"]
diff --git a/tests/test_integration.py b/tests/test_integration.py
deleted file mode 100644
index aa7b89ae..00000000
--- a/tests/test_integration.py
+++ /dev/null
@@ -1,758 +0,0 @@
-"""
-This module contains integration tests meant to run against a test Mastodon instance.
-
-You can set up a test instance locally by following this guide:
-https://docs.joinmastodon.org/dev/setup/
-
-To enable integration tests, export the following environment variables to match
-your test server and database:
-
-```
-export TOOT_TEST_HOSTNAME="localhost:3000"
-export TOOT_TEST_DATABASE_DSN="dbname=mastodon_development"
-```
-"""
-
-import os
-import psycopg2
-import pytest
-import re
-import time
-import uuid
-
-from datetime import datetime, timedelta, timezone
-from os import path
-from toot import CLIENT_NAME, CLIENT_WEBSITE, api, App, User
-from toot.console import run_command
-from toot.exceptions import ConsoleError, NotFoundError
-from toot.utils import get_text
-from unittest import mock
-
-# Host name of a test instance to run integration tests against
-# DO NOT USE PUBLIC INSTANCES!!!
-BASE_URL = os.getenv("TOOT_TEST_BASE_URL")
-
-# Mastodon database name, used to confirm user registration without having to click the link
-DATABASE_DSN = os.getenv("TOOT_TEST_DATABASE_DSN")
-
-# Toot logo used for testing image upload
-TRUMPET = path.join(path.dirname(path.dirname(path.realpath(__file__))), "trumpet.png")
-
-
-if not BASE_URL or not DATABASE_DSN:
-    pytest.skip("Skipping integration tests", allow_module_level=True)
-
-# ------------------------------------------------------------------------------
-# Fixtures
-# ------------------------------------------------------------------------------
-
-
-def create_app():
-    instance = api.get_instance(BASE_URL)
-    response = api.create_app(BASE_URL)
-    return App(instance["uri"], BASE_URL, response["client_id"], response["client_secret"])
-
-
-def register_account(app: App):
-    username = str(uuid.uuid4())[-10:]
-    email = f"{username}@example.com"
-
-    response = api.register_account(app, username, email, "password", "en")
-    confirm_user(email)
-    return User(app.instance, username, response["access_token"])
-
-
-def confirm_user(email):
-    conn = psycopg2.connect(DATABASE_DSN)
-    cursor = conn.cursor()
-    cursor.execute("UPDATE users SET confirmed_at = now() WHERE email = %s;", (email,))
-    conn.commit()
-
-
-@pytest.fixture(scope="session")
-def app():
-    return create_app()
-
-
-@pytest.fixture(scope="session")
-def user(app):
-    return register_account(app)
-
-
-@pytest.fixture(scope="session")
-def friend(app):
-    return register_account(app)
-
-
-@pytest.fixture
-def run(app, user, capsys):
-    def _run(command, *params, as_user=None):
-        run_command(app, as_user or user, command, params or [])
-        out, err = capsys.readouterr()
-        assert err == ""
-        return strip_ansi(out)
-    return _run
-
-
-@pytest.fixture
-def run_anon(capsys):
-    def _run(command, *params):
-        run_command(None, None, command, params or [])
-        out, err = capsys.readouterr()
-        assert err == ""
-        return strip_ansi(out)
-    return _run
-
-# ------------------------------------------------------------------------------
-# Tests
-# ------------------------------------------------------------------------------
-
-
-def test_instance(app, run):
-    out = run("instance", "--disable-https")
-    assert "Mastodon" in out
-    assert app.instance in out
-    assert "running Mastodon" in out
-
-
-def test_instance_anon(app, run_anon):
-    out = run_anon("instance", BASE_URL)
-    assert "Mastodon" in out
-    assert app.instance in out
-    assert "running Mastodon" in out
-
-    # Need to specify the instance name when running anon
-    with pytest.raises(ConsoleError) as exc:
-        run_anon("instance")
-    assert str(exc.value) == "Please specify an instance."
-
-
-def test_post(app, user, run):
-    text = "i wish i was a #lumberjack"
-    out = run("post", text)
-    status_id = _posted_status_id(out)
-
-    status = api.fetch_status(app, user, status_id)
-    assert text == get_text(status["content"])
-    assert status["visibility"] == "public"
-    assert status["sensitive"] is False
-    assert status["spoiler_text"] == ""
-    assert status["poll"] is None
-
-    # Pleroma doesn't return the application
-    if status["application"]:
-        assert status["application"]["name"] == CLIENT_NAME
-        assert status["application"]["website"] == CLIENT_WEBSITE
-
-
-def test_post_visibility(app, user, run):
-    for visibility in ["public", "unlisted", "private", "direct"]:
-        out = run("post", "foo", "--visibility", visibility)
-        status_id = _posted_status_id(out)
-        status = api.fetch_status(app, user, status_id)
-        assert status["visibility"] == visibility
-
-
-def test_post_scheduled_at(app, user, run):
-    text = str(uuid.uuid4())
-    scheduled_at = datetime.now(timezone.utc).replace(microsecond=0) + timedelta(minutes=10)
-
-    out = run("post", text, "--scheduled-at", scheduled_at.isoformat())
-    assert "Toot scheduled for" in out
-
-    statuses = api.scheduled_statuses(app, user)
-    [status] = [s for s in statuses if s["params"]["text"] == text]
-    assert datetime.strptime(status["scheduled_at"], "%Y-%m-%dT%H:%M:%S.%f%z") == scheduled_at
-
-
-def test_post_scheduled_in(app, user, run):
-    text = str(uuid.uuid4())
-
-    variants = [
-        ("1 day", timedelta(days=1)),
-        ("1 day 6 hours", timedelta(days=1, hours=6)),
-        ("1 day 6 hours 13 minutes", timedelta(days=1, hours=6, minutes=13)),
-        ("1 day 6 hours 13 minutes 51 second", timedelta(days=1, hours=6, minutes=13, seconds=51)),
-        ("2d", timedelta(days=2)),
-        ("2d6h", timedelta(days=2, hours=6)),
-        ("2d6h13m", timedelta(days=2, hours=6, minutes=13)),
-        ("2d6h13m51s", timedelta(days=2, hours=6, minutes=13, seconds=51)),
-    ]
-
-    datetimes = []
-    for scheduled_in, delta in variants:
-        out = run("post", text, "--scheduled-in", scheduled_in)
-        dttm = datetime.utcnow() + delta
-        assert out.startswith(f"Toot scheduled for: {str(dttm)[:16]}")
-        datetimes.append(dttm)
-
-    scheduled = api.scheduled_statuses(app, user)
-    scheduled = [s for s in scheduled if s["params"]["text"] == text]
-    scheduled = sorted(scheduled, key=lambda s: s["scheduled_at"])
-    assert len(scheduled) == 8
-
-    for expected, status in zip(datetimes, scheduled):
-        actual = datetime.strptime(status["scheduled_at"], "%Y-%m-%dT%H:%M:%S.%fZ")
-        delta = expected - actual
-        assert delta.total_seconds() < 5
-
-
-def test_post_poll(app, user, run):
-    text = str(uuid.uuid4())
-
-    out = run(
-        "post", text,
-        "--poll-option", "foo",
-        "--poll-option", "bar",
-        "--poll-option", "baz",
-        "--poll-option", "qux",
-    )
-
-    status_id = _posted_status_id(out)
-
-    status = api.fetch_status(app, user, status_id)
-    assert status["poll"]["expired"] is False
-    assert status["poll"]["multiple"] is False
-    assert status["poll"]["options"] == [
-        {"title": "foo", "votes_count": 0},
-        {"title": "bar", "votes_count": 0},
-        {"title": "baz", "votes_count": 0},
-        {"title": "qux", "votes_count": 0}
-    ]
-
-    # Test expires_at is 24h by default
-    actual = datetime.strptime(status["poll"]["expires_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
-    expected = datetime.now(timezone.utc) + timedelta(days=1)
-    delta = actual - expected
-    assert delta.total_seconds() < 5
-
-
-def test_post_poll_multiple(app, user, run):
-    text = str(uuid.uuid4())
-
-    out = run(
-        "post", text,
-        "--poll-option", "foo",
-        "--poll-option", "bar",
-        "--poll-multiple"
-    )
-
-    status_id = _posted_status_id(out)
-
-    status = api.fetch_status(app, user, status_id)
-    assert status["poll"]["multiple"] is True
-
-
-def test_post_poll_expires_in(app, user, run):
-    text = str(uuid.uuid4())
-
-    out = run(
-        "post", text,
-        "--poll-option", "foo",
-        "--poll-option", "bar",
-        "--poll-expires-in", "8h",
-    )
-
-    status_id = _posted_status_id(out)
-
-    status = api.fetch_status(app, user, status_id)
-    actual = datetime.strptime(status["poll"]["expires_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
-    expected = datetime.now(timezone.utc) + timedelta(hours=8)
-    delta = actual - expected
-    assert delta.total_seconds() < 5
-
-
-def test_post_poll_hide_totals(app, user, run):
-    text = str(uuid.uuid4())
-
-    out = run(
-        "post", text,
-        "--poll-option", "foo",
-        "--poll-option", "bar",
-        "--poll-hide-totals"
-    )
-
-    status_id = _posted_status_id(out)
-
-    status = api.fetch_status(app, user, status_id)
-
-    # votes_count is None when totals are hidden
-    assert status["poll"]["options"] == [
-        {"title": "foo", "votes_count": None},
-        {"title": "bar", "votes_count": None},
-    ]
-
-
-def test_post_language(app, user, run):
-    out = run("post", "test", "--language", "hr")
-    status_id = _posted_status_id(out)
-    status = api.fetch_status(app, user, status_id)
-    assert status["language"] == "hr"
-
-    out = run("post", "test", "--language", "zh")
-    status_id = _posted_status_id(out)
-    status = api.fetch_status(app, user, status_id)
-    assert status["language"] == "zh"
-
-
-def test_media_thumbnail(app, user, run):
-    assets_dir = path.realpath(path.join(path.dirname(__file__), "assets"))
-
-    video_path = path.join(assets_dir, "small.webm")
-    thumbnail_path = path.join(assets_dir, "test1.png")
-
-    out = run(
-        "post",
-        "--media", video_path,
-        "--thumbnail", thumbnail_path,
-        "--description", "foo",
-        "some text"
-    )
-
-    status_id = _posted_status_id(out)
-    status = api.fetch_status(app, user, status_id)
-    [media] = status["media_attachments"]
-
-    assert media["description"] == "foo"
-    assert media["type"] == "video"
-    assert media["url"].endswith(".mp4")
-    assert media["preview_url"].endswith(".png")
-
-    # Video properties
-    assert int(media["meta"]["original"]["duration"]) == 5
-    assert media["meta"]["original"]["height"] == 320
-    assert media["meta"]["original"]["width"] == 560
-
-    # Thumbnail properties
-    assert media["meta"]["small"]["height"] == 50
-    assert media["meta"]["small"]["width"] == 50
-
-
-def test_media_attachments(app, user, run):
-    assets_dir = path.realpath(path.join(path.dirname(__file__), "assets"))
-
-    path1 = path.join(assets_dir, "test1.png")
-    path2 = path.join(assets_dir, "test2.png")
-    path3 = path.join(assets_dir, "test3.png")
-    path4 = path.join(assets_dir, "test4.png")
-
-    out = run(
-        "post",
-        "--media", path1,
-        "--media", path2,
-        "--media", path3,
-        "--media", path4,
-        "--description", "Test 1",
-        "--description", "Test 2",
-        "--description", "Test 3",
-        "--description", "Test 4",
-        "some text"
-    )
-
-    status_id = _posted_status_id(out)
-    status = api.fetch_status(app, user, status_id)
-
-    [a1, a2, a3, a4] = status["media_attachments"]
-
-    # Pleroma doesn't send metadata
-    if "meta" in a1:
-        assert a1["meta"]["original"]["size"] == "50x50"
-        assert a2["meta"]["original"]["size"] == "50x60"
-        assert a3["meta"]["original"]["size"] == "50x70"
-        assert a4["meta"]["original"]["size"] == "50x80"
-
-    assert a1["description"] == "Test 1"
-    assert a2["description"] == "Test 2"
-    assert a3["description"] == "Test 3"
-    assert a4["description"] == "Test 4"
-
-
-@mock.patch("toot.utils.multiline_input")
-@mock.patch("sys.stdin.read")
-def test_media_attachment_without_text(mock_read, mock_ml, app, user, run):
-    # No status from stdin or readline
-    mock_read.return_value = ""
-    mock_ml.return_value = ""
-
-    assets_dir = path.realpath(path.join(path.dirname(__file__), "assets"))
-    media_path = path.join(assets_dir, "test1.png")
-
-    out = run("post", "--media", media_path)
-    status_id = _posted_status_id(out)
-
-    status = api.fetch_status(app, user, status_id)
-    assert status["content"] == ""
-
-    [attachment] = status["media_attachments"]
-    assert not attachment["description"]
-
-    # Pleroma doesn't send metadata
-    if "meta" in attachment:
-        assert attachment["meta"]["original"]["size"] == "50x50"
-
-
-def test_delete_status(app, user, run):
-    status = api.post_status(app, user, "foo")
-
-    out = run("delete", status["id"])
-    assert out == "✓ Status deleted"
-
-    with pytest.raises(NotFoundError):
-        api.fetch_status(app, user, status["id"])
-
-
-def test_reply_thread(app, user, friend, run):
-    status = api.post_status(app, friend, "This is the status")
-
-    out = run("post", "--reply-to", status["id"], "This is the reply")
-    status_id = _posted_status_id(out)
-    reply = api.fetch_status(app, user, status_id)
-
-    assert reply["in_reply_to_id"] == status["id"]
-
-    out = run("thread", status["id"])
-    [s1, s2] = [s.strip() for s in re.split(r"─+", out) if s.strip()]
-
-    assert "This is the status" in s1
-    assert "This is the reply" in s2
-    assert friend.username in s1
-    assert user.username in s2
-    assert status["id"] in s1
-    assert reply["id"] in s2
-
-
-def test_favourite(app, user, run):
-    status = api.post_status(app, user, "foo")
-    assert not status["favourited"]
-
-    out = run("favourite", status["id"])
-    assert out == "✓ Status favourited"
-
-    status = api.fetch_status(app, user, status["id"])
-    assert status["favourited"]
-
-    out = run("unfavourite", status["id"])
-    assert out == "✓ Status unfavourited"
-
-    # A short delay is required before the server returns new data
-    time.sleep(0.1)
-
-    status = api.fetch_status(app, user, status["id"])
-    assert not status["favourited"]
-
-
-def test_reblog(app, user, run):
-    status = api.post_status(app, user, "foo")
-    assert not status["reblogged"]
-
-    out = run("reblog", status["id"])
-    assert out == "✓ Status reblogged"
-
-    status = api.fetch_status(app, user, status["id"])
-    assert status["reblogged"]
-
-    out = run("reblogged_by", status["id"])
-    assert out == f"@{user.username}"
-
-    out = run("unreblog", status["id"])
-    assert out == "✓ Status unreblogged"
-
-    status = api.fetch_status(app, user, status["id"])
-    assert not status["reblogged"]
-
-
-def test_pin(app, user, run):
-    status = api.post_status(app, user, "foo")
-    assert not status["pinned"]
-
-    out = run("pin", status["id"])
-    assert out == "✓ Status pinned"
-
-    status = api.fetch_status(app, user, status["id"])
-    assert status["pinned"]
-
-    out = run("unpin", status["id"])
-    assert out == "✓ Status unpinned"
-
-    status = api.fetch_status(app, user, status["id"])
-    assert not status["pinned"]
-
-
-def test_bookmark(app, user, run):
-    status = api.post_status(app, user, "foo")
-    assert not status["bookmarked"]
-
-    out = run("bookmark", status["id"])
-    assert out == "✓ Status bookmarked"
-
-    status = api.fetch_status(app, user, status["id"])
-    assert status["bookmarked"]
-
-    out = run("unbookmark", status["id"])
-    assert out == "✓ Status unbookmarked"
-
-    status = api.fetch_status(app, user, status["id"])
-    assert not status["bookmarked"]
-
-
-def test_whoami(user, run):
-    out = run("whoami")
-    # TODO: test other fields once updating account is supported
-    assert f"@{user.username}" in out
-
-
-def test_whois(app, friend, run):
-    variants = [
-        friend.username,
-        f"@{friend.username}",
-        f"{friend.username}@{app.instance}",
-        f"@{friend.username}@{app.instance}",
-    ]
-
-    for username in variants:
-        out = run("whois", username)
-        assert f"@{friend.username}" in out
-
-
-def test_search_account(friend, run):
-    out = run("search", friend.username)
-    assert out == f"Accounts:\n* @{friend.username}"
-
-
-def test_search_hashtag(app, user, run):
-    api.post_status(app, user, "#hashtag_x")
-    api.post_status(app, user, "#hashtag_y")
-    api.post_status(app, user, "#hashtag_z")
-
-    out = run("search", "#hashtag")
-    assert out == "Hashtags:\n#hashtag_x, #hashtag_y, #hashtag_z"
-
-
-def test_follow(friend, run):
-    out = run("follow", friend.username)
-    assert out == f"✓ You are now following {friend.username}"
-
-    out = run("unfollow", friend.username)
-    assert out == f"✓ You are no longer following {friend.username}"
-
-
-def test_follow_case_insensitive(friend, run):
-    username = friend.username.upper()
-
-    out = run("follow", username)
-    assert out == f"✓ You are now following {username}"
-
-    out = run("unfollow", username)
-    assert out == f"✓ You are no longer following {username}"
-
-
-# TODO: improve testing stderr, catching exceptions is not optimal
-def test_follow_not_found(run):
-    with pytest.raises(ConsoleError) as ex_info:
-        run("follow", "banana")
-    assert str(ex_info.value) == "Account not found"
-
-
-def test_mute(app, user, friend, run):
-    out = run("mute", friend.username)
-    assert out == f"✓ You have muted {friend.username}"
-
-    [muted_account] = api.get_muted_accounts(app, user)
-    assert muted_account["acct"] == friend.username
-
-    out = run("unmute", friend.username)
-    assert out == f"✓ {friend.username} is no longer muted"
-
-    assert api.get_muted_accounts(app, user) == []
-
-
-def test_block(app, user, friend, run):
-    out = run("block", friend.username)
-    assert out == f"✓ You are now blocking {friend.username}"
-
-    [blockd_account] = api.get_blocked_accounts(app, user)
-    assert blockd_account["acct"] == friend.username
-
-    out = run("unblock", friend.username)
-    assert out == f"✓ {friend.username} is no longer blocked"
-
-    assert api.get_blocked_accounts(app, user) == []
-
-
-def test_following_followers(user, friend, run):
-    out = run("following", user.username)
-    assert out == ""
-
-    run("follow", friend.username)
-
-    out = run("following", user.username)
-    assert out == f"* @{friend.username}"
-
-    out = run("followers", friend.username)
-    assert out == f"* @{user.username}"
-
-
-def test_tags(run):
-    out = run("tags_followed")
-    assert out == "You're not following any hashtags."
-
-    out = run("tags_follow", "foo")
-    assert out == "✓ You are now following #foo"
-
-    out = run("tags_followed")
-    assert out == f"* #foo\t{BASE_URL}/tags/foo"
-
-    out = run("tags_follow", "bar")
-    assert out == "✓ You are now following #bar"
-
-    out = run("tags_followed")
-    assert out == "\n".join([
-        f"* #bar\t{BASE_URL}/tags/bar",
-        f"* #foo\t{BASE_URL}/tags/foo",
-    ])
-
-    out = run("tags_unfollow", "foo")
-    assert out == "✓ You are no longer following #foo"
-
-    out = run("tags_followed")
-    assert out == f"* #bar\t{BASE_URL}/tags/bar"
-
-
-def test_update_account_no_options(run):
-    with pytest.raises(ConsoleError) as exc:
-        run("update_account")
-    assert str(exc.value) == "Please specify at least one option to update the account"
-
-
-def test_update_account_display_name(run, app, user):
-    out = run("update_account", "--display-name", "elwood")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["display_name"] == "elwood"
-
-
-def test_update_account_note(run, app, user):
-    note = ("It's 106 miles to Chicago, we got a full tank of gas, half a pack "
-           "of cigarettes, it's dark... and we're wearing sunglasses.")
-
-    out = run("update_account", "--note", note)
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert get_text(account["note"]) == note
-
-
-def test_update_account_language(run, app, user):
-    out = run("update_account", "--language", "hr")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["source"]["language"] == "hr"
-
-
-def test_update_account_privacy(run, app, user):
-    out = run("update_account", "--privacy", "private")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["source"]["privacy"] == "private"
-
-
-def test_update_account_avatar(run, app, user):
-    account = api.verify_credentials(app, user)
-    old_value = account["avatar"]
-
-    out = run("update_account", "--avatar", TRUMPET)
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["avatar"] != old_value
-
-
-def test_update_account_header(run, app, user):
-    account = api.verify_credentials(app, user)
-    old_value = account["header"]
-
-    out = run("update_account", "--header", TRUMPET)
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["header"] != old_value
-
-
-def test_update_account_locked(run, app, user):
-    out = run("update_account", "--locked")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["locked"] is True
-
-    out = run("update_account", "--no-locked")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["locked"] is False
-
-
-def test_update_account_bot(run, app, user):
-    out = run("update_account", "--bot")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["bot"] is True
-
-    out = run("update_account", "--no-bot")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["bot"] is False
-
-
-def test_update_account_discoverable(run, app, user):
-    out = run("update_account", "--discoverable")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["discoverable"] is True
-
-    out = run("update_account", "--no-discoverable")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["discoverable"] is False
-
-
-def test_update_account_sensitive(run, app, user):
-    out = run("update_account", "--sensitive")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["source"]["sensitive"] is True
-
-    out = run("update_account", "--no-sensitive")
-    assert out == "✓ Account updated"
-
-    account = api.verify_credentials(app, user)
-    assert account["source"]["sensitive"] is False
-
-
-# ------------------------------------------------------------------------------
-# Utils
-# ------------------------------------------------------------------------------
-
-strip_ansi_pattern = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
-
-
-def strip_ansi(string):
-    return strip_ansi_pattern.sub("", string).strip()
-
-
-def _posted_status_id(out):
-    pattern = re.compile(r"Toot posted: http://([^/]+)/([^/]+)/(.+)")
-    match = re.search(pattern, out)
-    assert match
-
-    _, _, status_id = match.groups()
-
-    return status_id

From 1c9935effcd67e2feada1fcbe04b0591808967be Mon Sep 17 00:00:00 2001
From: Ivan Habunek <ivan@habunek.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 18/55] Add a simple table printer and apply to lists

---
 toot/commands.py | 14 +++++++++-----
 toot/output.py   | 39 +++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/toot/commands.py b/toot/commands.py
index a2879e94..ce34e589 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -7,9 +7,9 @@
 from toot import api, config, __version__
 from toot.auth import login_interactive, login_browser_interactive, create_app_interactive
 from toot.exceptions import ApiError, ConsoleError
-from toot.output import (print_out, print_instance, print_account, print_acct_list,
-                         print_search_results, print_timeline, print_notifications,
-                         print_tag_list, print_list_list, print_list_accounts)
+from toot.output import (print_lists, print_out, print_instance, print_account, print_acct_list,
+                         print_search_results, print_table, print_timeline, print_notifications,
+                         print_tag_list, print_list_accounts)
 from toot.tui.utils import parse_datetime
 from toot.utils import args_get_instance, delete_tmp_status_file, editor_input, multiline_input, EOF_KEY
 
@@ -425,8 +425,12 @@ def tags_followed(app, user, args):
 
 
 def lists(app, user, args):
-    response = api.get_lists(app, user)
-    print_list_list(response)
+    lists = api.get_lists(app, user)
+
+    if lists:
+        print_lists(lists)
+    else:
+        print_out("You have no lists defined.")
 
 
 def list_accounts(app, user, args):
diff --git a/toot/output.py b/toot/output.py
index 25f72529..8015b7b2 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -3,9 +3,10 @@
 import sys
 import textwrap
 
-from toot.tui.utils import parse_datetime
+from typing import List
 from wcwidth import wcswidth
 
+from toot.tui.utils import parse_datetime
 from toot.utils import get_text, parse_html
 from toot.wcstring import wc_wrap
 
@@ -210,15 +211,33 @@ def print_tag_list(tags):
         print_out("You're not following any hashtags.")
 
 
-def print_list_list(lists):
-    if lists:
-        for list_item in lists:
-            replies_policy = list_item['replies_policy'] if list_item['replies_policy'] else ''
-            print_out(f"Title: <green>\"{list_item['title']}\"</green>\t"
-                      + f"ID: <green>{list_item['id']}\t</green>"
-                      + f"Replies policy: <green>{replies_policy}</green>")
-    else:
-        print_out("You have no lists defined.")
+def print_lists(lists):
+    headers = ["ID", "Title", "Replies"]
+    data = [[lst["id"], lst["title"], lst["replies_policy"]] for lst in lists]
+    print_table(headers, data)
+
+
+def print_table(headers: List[str], data: List[List[str]]):
+    widths = [[len(cell) for cell in row] for row in data + [headers]]
+    widths = [max(width) for width in zip(*widths)]
+
+    def style(string, tag):
+        return f"<{tag}>{string}</{tag}>" if tag else string
+
+    def print_row(row, tag=None):
+        for idx, cell in enumerate(row):
+            width = widths[idx]
+            print_out(style(cell.ljust(width), tag), end="")
+            print_out("  ", end="")
+        print_out()
+
+    underlines = ["-" * width for width in widths]
+
+    print_row(headers, "bold")
+    print_row(underlines, "dim")
+
+    for row in data:
+        print_row(row)
 
 
 def print_list_accounts(accounts):

From 4023bf07577ba72ff95bc8f819f12440d5507feb Mon Sep 17 00:00:00 2001
From: Ivan Habunek <ivan@habunek.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 19/55] Add integration tests for lists

---
 tests/integration/test_lists.py | 58 +++++++++++++++++++++++++++++++++
 toot/output.py                  |  2 +-
 2 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/test_lists.py

diff --git a/tests/integration/test_lists.py b/tests/integration/test_lists.py
new file mode 100644
index 00000000..c9fd392e
--- /dev/null
+++ b/tests/integration/test_lists.py
@@ -0,0 +1,58 @@
+
+from toot import api
+from tests.integration.conftest import register_account
+
+
+def test_lists_empty(run):
+    out = run("lists")
+    assert out == "You have no lists defined."
+
+
+def test_list_create_delete(run):
+    out = run("list_create", "banana")
+    assert out == '✓ List "banana" created.'
+
+    out = run("lists")
+    assert "banana" in out
+
+    out = run("list_create", "mango")
+    assert out == '✓ List "mango" created.'
+
+    out = run("lists")
+    assert "banana" in out
+    assert "mango" in out
+
+    out = run("list_delete", "banana")
+    assert out == '✓ List "banana" deleted.'
+
+    out = run("lists")
+    assert "banana" not in out
+    assert "mango" in out
+
+    out = run("list_delete", "mango")
+    assert out == '✓ List "mango" deleted.'
+
+    out = run("lists")
+    assert out == "You have no lists defined."
+
+
+def test_list_add_remove(run, app):
+    acc = register_account(app)
+    run("list_create", "foo")
+
+    out = run("list_add", "foo", acc.username)
+    assert out == f"You must follow @{acc.username} before adding this account to a list."
+
+    run("follow", acc.username)
+
+    out = run("list_add", "foo", acc.username)
+    assert out == f'✓ Added account "{acc.username}"'
+
+    out = run("list_accounts", "foo")
+    assert acc.username in out
+
+    out = run("list_remove", "foo", acc.username)
+    assert out == f'✓ Removed account "{acc.username}"'
+
+    out = run("list_accounts", "foo")
+    assert out == "This list has no accounts."
diff --git a/toot/output.py b/toot/output.py
index 8015b7b2..c89aca99 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -241,8 +241,8 @@ def print_row(row, tag=None):
 
 
 def print_list_accounts(accounts):
-    print_out("Accounts in list</green>:\n")
     if accounts:
+        print_out("Accounts in list</green>:\n")
         print_acct_list(accounts)
     else:
         print_out("This list has no accounts.")

From fe3044fafa947bfb521f2998d29267fc8988262b Mon Sep 17 00:00:00 2001
From: Ivan Habunek <ivan@habunek.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 20/55] Fix tests

---
 tests/integration/test_status.py | 2 +-
 tests/test_console.py            | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_status.py b/tests/integration/test_status.py
index 82741740..b23f44ea 100644
--- a/tests/integration/test_status.py
+++ b/tests/integration/test_status.py
@@ -46,7 +46,7 @@ def test_reblog(app, user, run):
     assert status["reblogged"]
 
     out = run("reblogged_by", status["id"])
-    assert out == f"@{user.username}"
+    assert user.username in out
 
     out = run("unreblog", status["id"])
     assert out == "✓ Status unreblogged"
diff --git a/tests/test_console.py b/tests/test_console.py
index 3b58d182..ffe1d12c 100644
--- a/tests/test_console.py
+++ b/tests/test_console.py
@@ -177,7 +177,7 @@ def test_timeline_with_re(mock_get, monkeypatch, capsys):
     mock_get.assert_called_once_with(app, user, '/api/v1/timelines/home', {'limit': 10})
 
     out, err = capsys.readouterr()
-    lines = out.split("\n")
+    lines = uncolorize(out).split("\n")
 
     assert "Frank Zappa" in lines[1]
     assert "@fz" in lines[1]
@@ -349,6 +349,7 @@ def test_search(mock_get, capsys):
     })
 
     out, err = capsys.readouterr()
+    out = uncolorize(out)
     assert "Hashtags:\n#foo, #bar, #baz" in out
     assert "Accounts:" in out
     assert "@thequeen Freddy Mercury" in out

From 006ad85971bef684ccf73374a645a5cefbc3429c Mon Sep 17 00:00:00 2001
From: Ivan Habunek <ivan@habunek.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 21/55] Extract fetching list ID

Also don't check if account is found, that function alredy raises a
ConsoleError.
---
 tests/integration/test_lists.py | 17 ++++++++++++++-
 toot/commands.py                | 37 ++++++++++++---------------------
 2 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/tests/integration/test_lists.py b/tests/integration/test_lists.py
index c9fd392e..dc2e3f52 100644
--- a/tests/integration/test_lists.py
+++ b/tests/integration/test_lists.py
@@ -1,6 +1,7 @@
 
-from toot import api
+import pytest
 from tests.integration.conftest import register_account
+from toot.exceptions import ConsoleError
 
 
 def test_lists_empty(run):
@@ -35,6 +36,10 @@ def test_list_create_delete(run):
     out = run("lists")
     assert out == "You have no lists defined."
 
+    with pytest.raises(ConsoleError) as exc:
+        run("list_delete", "mango")
+    assert str(exc.value) == "List not found"
+
 
 def test_list_add_remove(run, app):
     acc = register_account(app)
@@ -51,6 +56,16 @@ def test_list_add_remove(run, app):
     out = run("list_accounts", "foo")
     assert acc.username in out
 
+    # Account doesn't exist
+    with pytest.raises(ConsoleError) as exc:
+        run("list_add", "foo", "does_not_exist")
+    assert str(exc.value) == "Account not found"
+
+    # List doesn't exist
+    with pytest.raises(ConsoleError) as exc:
+        run("list_add", "does_not_exist", acc.username)
+    assert str(exc.value) == "List not found"
+
     out = run("list_remove", "foo", acc.username)
     assert out == f'✓ Removed account "{acc.username}"'
 
diff --git a/toot/commands.py b/toot/commands.py
index ce34e589..b17cf72b 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -434,11 +434,7 @@ def lists(app, user, args):
 
 
 def list_accounts(app, user, args):
-    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
-    if not list_id:
-        print_out("<red>List not found</red>")
-        return
-
+    list_id = _get_list_id(app, user, args)
     response = api.get_list_accounts(app, user, list_id)
     print_list_accounts(response)
 
@@ -449,24 +445,15 @@ def list_create(app, user, args):
 
 
 def list_delete(app, user, args):
-    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
-    if not list_id:
-        print_out("<red>List not found</red>")
-        return
-
+    list_id = _get_list_id(app, user, args)
     api.delete_list(app, user, list_id)
     print_out(f"<green>✓ List \"{args.title if args.title else args.id}\"</green> <red>deleted.</red>")
 
 
 def list_add(app, user, args):
-    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
-    if not list_id:
-        print_out("<red>List not found</red>")
-        return
+    list_id = _get_list_id(app, user, args)
     account = find_account(app, user, args.account)
-    if not account:
-        print_out("<red>Account not found</red>")
-        return
+
     try:
         api.add_accounts_to_list(app, user, list_id, [account['id']])
     except Exception as ex:
@@ -484,22 +471,24 @@ def list_add(app, user, args):
         else:
             print_out(f"<red>{ex}</red>")
         return
+
     print_out(f"<green>✓ Added account \"{args.account}\"</green>")
 
 
 def list_remove(app, user, args):
-    list_id = args.id if args.id else api.find_list_id(app, user, args.title)
-    if not list_id:
-        print_out("<red>List not found</red>")
-        return
+    list_id = _get_list_id(app, user, args)
     account = find_account(app, user, args.account)
-    if not account:
-        print_out("<red>Account not found</red>")
-        return
     api.remove_accounts_from_list(app, user, list_id, [account['id']])
     print_out(f"<green>✓ Removed account \"{args.account}\"</green>")
 
 
+def _get_list_id(app, user, args):
+    list_id = args.id or api.find_list_id(app, user, args.title)
+    if not list_id:
+        raise ConsoleError("List not found")
+    return list_id
+
+
 def mute(app, user, args):
     account = find_account(app, user, args.account)
     api.mute(app, user, account['id'])

From d3ea1e5db29f6e8ba59a1aec3dff3254f73adb27 Mon Sep 17 00:00:00 2001
From: Ivan Habunek <ivan@habunek.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 22/55] Simplify integration tests by catching ConsoleError

---
 tests/integration/conftest.py   | 10 +++++++++-
 tests/integration/test_auth.py  |  8 ++------
 tests/integration/test_lists.py | 18 ++++++------------
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index b4aaa1ef..ed3033a1 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -22,6 +22,8 @@
 from pathlib import Path
 from toot import api, App, User
 from toot.console import run_command
+from toot.exceptions import ApiError, ConsoleError
+from toot.output import print_out
 
 # Host name of a test instance to run integration tests against
 # DO NOT USE PUBLIC INSTANCES!!!
@@ -84,7 +86,13 @@ def friend(app):
 @pytest.fixture
 def run(app, user, capsys):
     def _run(command, *params, as_user=None):
-        run_command(app, as_user or user, command, params or [])
+        # The try/catch duplicates logic from console.main to convert exceptions
+        # to printed error messages. TODO: could be deduped
+        try:
+            run_command(app, as_user or user, command, params or [])
+        except (ConsoleError, ApiError) as e:
+            print_out(str(e))
+
         out, err = capsys.readouterr()
         assert err == ""
         return strip_ansi(out)
diff --git a/tests/integration/test_auth.py b/tests/integration/test_auth.py
index afe5c39d..c786c4b4 100644
--- a/tests/integration/test_auth.py
+++ b/tests/integration/test_auth.py
@@ -1,15 +1,11 @@
-import pytest
-
 from tests.integration.conftest import TRUMPET
 from toot import api
-from toot.exceptions import ConsoleError
 from toot.utils import get_text
 
 
 def test_update_account_no_options(run):
-    with pytest.raises(ConsoleError) as exc:
-        run("update_account")
-    assert str(exc.value) == "Please specify at least one option to update the account"
+    out = run("update_account")
+    assert out == "Please specify at least one option to update the account"
 
 
 def test_update_account_display_name(run, app, user):
diff --git a/tests/integration/test_lists.py b/tests/integration/test_lists.py
index dc2e3f52..6f98998d 100644
--- a/tests/integration/test_lists.py
+++ b/tests/integration/test_lists.py
@@ -1,7 +1,4 @@
-
-import pytest
 from tests.integration.conftest import register_account
-from toot.exceptions import ConsoleError
 
 
 def test_lists_empty(run):
@@ -36,9 +33,8 @@ def test_list_create_delete(run):
     out = run("lists")
     assert out == "You have no lists defined."
 
-    with pytest.raises(ConsoleError) as exc:
-        run("list_delete", "mango")
-    assert str(exc.value) == "List not found"
+    out = run("list_delete", "mango")
+    assert out == "List not found"
 
 
 def test_list_add_remove(run, app):
@@ -57,14 +53,12 @@ def test_list_add_remove(run, app):
     assert acc.username in out
 
     # Account doesn't exist
-    with pytest.raises(ConsoleError) as exc:
-        run("list_add", "foo", "does_not_exist")
-    assert str(exc.value) == "Account not found"
+    out = run("list_add", "foo", "does_not_exist")
+    assert out == "Account not found"
 
     # List doesn't exist
-    with pytest.raises(ConsoleError) as exc:
-        run("list_add", "does_not_exist", acc.username)
-    assert str(exc.value) == "List not found"
+    out = run("list_add", "does_not_exist", acc.username)
+    assert out == "List not found"
 
     out = run("list_remove", "foo", acc.username)
     assert out == f'✓ Removed account "{acc.username}"'

From 1fb05616e2165a38dd580129e535f089b5935f9c Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 23/55] Slightly tightened up list number spacing

---
 Makefile              |  2 +-
 toot/tui/constants.py | 12 ++++++------
 toot/tui/richtext.py  |  8 +++++---
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index cafe213b..e46a7399 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ publish :
 test:
 	pytest -v
 	flake8
-	vermin --target=3.6 --no-tips --violations --exclude-regex venv/.* .
+	vermin --target=3.6 --no-tips --violations --exclude-regex 'venv/.*' .
 
 coverage:
 	coverage erase
diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index 13f201dd..23450017 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -60,12 +60,12 @@
 
     ('blockquote', 'light gray', ''),
 
-    ('h1', 'yellow, bold', ''),
-    ('h2', 'dark red, bold', ''),
-    ('h3', 'yellow, bold', ''),
-    ('h4', 'yellow, bold', ''),
-    ('h5', 'yellow, bold', ''),
-    ('h6', 'yellow, bold', ''),
+    ('h1', 'white, bold', ''),
+    ('h2', 'white, bold', ''),
+    ('h3', 'white, bold', ''),
+    ('h4', 'white, bold', ''),
+    ('h5', 'white, bold', ''),
+    ('h6', 'white, bold', ''),
 
     ('class_mention_hashtag', 'light cyan,bold', ''),
 
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index d361f909..a40f6495 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -110,12 +110,14 @@ def get_style_name(self, tag) -> str:
         # used in anchor tags
         # Classes to blacklist: "invisible" used in Akkoma
         # anchor titles
-        style_name = tag.name
+
         if "class" in tag.attrs:
             clss = tag.attrs["class"]
             if len(clss) > 0:
                 style_name = "class_" + "_".join(clss)
-        return style_name
+                return style_name
+
+        style_name = tag.name
 
     # Tag handlers start here.
     # Tags not explicitly listed are "supported" by
@@ -244,7 +246,7 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
                     txt = urwid.Text(("li", "*"))
 
                 columns = urwid.Columns(
-                    [txt, ("weight", 9999, markup)], dividechars=1, min_width=4
+                    [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
                 )
                 widgets.append(columns)
             i += 1

From c48501e7ce7f628579ea9e98b93e40820973579f Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 24/55] renamed get_style_name to get_urwid_attr_name for
 clarity

---
 toot/tui/constants.py | 8 --------
 toot/tui/richtext.py  | 8 ++++----
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index 23450017..d308070a 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -40,33 +40,25 @@
     ('white_bold', 'white,bold', ''),
 
     # HTML tag styling
-
     # note, anchor styling is often overridden
     # by class names in Mastodon statuses
     # so you won't see the italics.
     ('a', ',italics', ''),
     ('em', 'white,italics', ''),
     ('i', 'white,italics', ''),
-
     ('strong', 'white,bold', ''),
     ('b', 'white,bold', ''),
-
     ('u', 'white,underline', ''),
-
     ('del', 'white, strikethrough', ''),
-
     ('code', 'white, standout', ''),
     ('pre', 'white, standout', ''),
-
     ('blockquote', 'light gray', ''),
-
     ('h1', 'white, bold', ''),
     ('h2', 'white, bold', ''),
     ('h3', 'white, bold', ''),
     ('h4', 'white, bold', ''),
     ('h5', 'white, bold', ''),
     ('h6', 'white, bold', ''),
-
     ('class_mention_hashtag', 'light cyan,bold', ''),
 
 ]
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index a40f6495..3cb842d3 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -100,7 +100,7 @@ def process_block_tag_children(self, tag) -> List[urwid.Widget]:
 
         return widget_list
 
-    def get_style_name(self, tag) -> str:
+    def get_urwid_attr_name(self, tag) -> str:
         """Get the class name and translate to a
         name suitable for use as an urwid
         text attribute name"""
@@ -139,7 +139,7 @@ def _a(self, tag) -> list:
         # in that case; set this up in constants.py
         # to control highlighting of hashtags
 
-        return (self.get_style_name(tag), markups)
+        return (self.get_urwid_attr_name(tag), markups)
 
     def _blockquote(self, tag) -> urwid.Widget:
         widget_list = self.process_block_tag_children(tag)
@@ -211,9 +211,9 @@ def _span(self, tag) -> list:
         # of its own
 
         if "class" in tag.attrs:
-            style_name = self.get_style_name(tag)
+            style_name = self.get_urwid_attr_name(tag)
         elif tag.parent:
-            style_name = self.get_style_name(tag.parent)
+            style_name = self.get_urwid_attr_name(tag.parent)
         else:
             style_name = tag.name
 

From dc9fa05d7a44f1926869116f64390be697e52be2 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 25/55] Make output match existing master branch status
 rendering exactly

Top level widgets are separated by blank lines, but
The final blank line of the status is omitted. This exactly
matches existing status rendering in master, for statuses that
contain only the currently supported tags
---
 toot/tui/richtext.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 3cb842d3..4e0aae68 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -34,7 +34,9 @@ def html_to_widgets(self, html) -> List[urwid.Widget]:
                     min_width=None,
                 )
             widgets.append(markup)
-        return widgets
+            # separate top level widgets with a blank line
+            widgets.append(urwid.Divider(" "))
+        return widgets[:-1]  # but suppress the last blank line
 
     def inline_tag_to_text(self, tag) -> list:
         """Convert html tag to plain text with tag as attributes recursively"""

From 9ed0a47a2b0b8339f16ce0d8ec72601c1c98cf33 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 26/55] fixed up palette constants a bit

No longer specifying color "white" when it's more correct to
omit the color and just specify an attribute like underline,
bold, etc.
---
 toot/tui/constants.py | 28 ++++++++++++++--------------
 toot/tui/richtext.py  |  4 ++--
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index d308070a..e2c2fd0a 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -44,21 +44,21 @@
     # by class names in Mastodon statuses
     # so you won't see the italics.
     ('a', ',italics', ''),
-    ('em', 'white,italics', ''),
-    ('i', 'white,italics', ''),
-    ('strong', 'white,bold', ''),
-    ('b', 'white,bold', ''),
-    ('u', 'white,underline', ''),
-    ('del', 'white, strikethrough', ''),
-    ('code', 'white, standout', ''),
-    ('pre', 'white, standout', ''),
+    ('em', ',italics', ''),
+    ('i', ',italics', ''),
+    ('strong', ',bold', ''),
+    ('b', ',bold', ''),
+    ('u', ',underline', ''),
+    ('del', ',strikethrough', ''),
+    ('code', 'dark gray, standout', ''),
+    ('pre', 'dark gray, standout', ''),
     ('blockquote', 'light gray', ''),
-    ('h1', 'white, bold', ''),
-    ('h2', 'white, bold', ''),
-    ('h3', 'white, bold', ''),
-    ('h4', 'white, bold', ''),
-    ('h5', 'white, bold', ''),
-    ('h6', 'white, bold', ''),
+    ('h1', ',bold', ''),
+    ('h2', ',bold', ''),
+    ('h3', ',bold', ''),
+    ('h4', ',bold', ''),
+    ('h5', ',bold', ''),
+    ('h6', ',bold', ''),
     ('class_mention_hashtag', 'light cyan,bold', ''),
 
 ]
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 4e0aae68..e41433c1 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -239,13 +239,13 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
                         ("li", [str(i), ". ", markup])
                     )  # 1. foo, 2. bar, etc.
                 else:
-                    txt = urwid.Text(("li", ["* ", markup]))  # * foo, * bar, etc.
+                    txt = urwid.Text(("li", ["\N{bullet} ", markup]))  # * foo, * bar, etc.
                 widgets.append(txt)
             else:
                 if ordered:
                     txt = urwid.Text(("li", [str(i) + "."]))
                 else:
-                    txt = urwid.Text(("li", "*"))
+                    txt = urwid.Text(("li", "\N{bullet}"))
 
                 columns = urwid.Columns(
                     [txt, ("weight", 9999, markup)], dividechars=1, min_width=3

From 6e1d36fe15180946aedfacfbe26cc4b538184db8 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 27/55] Removed unneeded import

---
 toot/commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/toot/commands.py b/toot/commands.py
index b17cf72b..58391dca 100644
--- a/toot/commands.py
+++ b/toot/commands.py
@@ -8,7 +8,7 @@
 from toot.auth import login_interactive, login_browser_interactive, create_app_interactive
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import (print_lists, print_out, print_instance, print_account, print_acct_list,
-                         print_search_results, print_table, print_timeline, print_notifications,
+                         print_search_results, print_timeline, print_notifications,
                          print_tag_list, print_list_accounts)
 from toot.tui.utils import parse_datetime
 from toot.utils import args_get_instance, delete_tmp_status_file, editor_input, multiline_input, EOF_KEY

From 0098371b0ef0c1ae09cb99e30ab65644c07c947f Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 28/55] Unicode normalize 'NKFC' incoming HTML text before
 rendering

---
 toot/tui/richtext.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index e41433c1..ac9685d6 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -3,6 +3,7 @@
 """
 from typing import List
 import urwid
+import unicodedata
 from bs4 import BeautifulSoup
 from bs4.element import NavigableString, Tag
 
@@ -14,6 +15,7 @@ def __init__(self, config={}):
     def html_to_widgets(self, html) -> List[urwid.Widget]:
         """Convert html to urwid widgets"""
         widgets: List[urwid.Widget] = []
+        html = unicodedata.normalize('NFKC', html)
         soup = BeautifulSoup(html.replace("&apos;", "'"), "html.parser")
         for e in soup.body or soup:
             if isinstance(e, NavigableString):

From 84663d6dcab97684a788bdaee62779b4bb8a5d37 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 29/55] Cleaned up type violations

---
 toot/tui/richtext.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index ac9685d6..225d52ee 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -1,7 +1,7 @@
 """
 richtext
 """
-from typing import List
+from typing import List, Tuple
 import urwid
 import unicodedata
 from bs4 import BeautifulSoup
@@ -9,7 +9,7 @@
 
 
 class ContentParser:
-    def __init__(self, config={}):
+    def __init__(self):
         """Parse a limited subset of HTML and create urwid widgets."""
 
     def html_to_widgets(self, html) -> List[urwid.Widget]:
@@ -40,14 +40,14 @@ def html_to_widgets(self, html) -> List[urwid.Widget]:
             widgets.append(urwid.Divider(" "))
         return widgets[:-1]  # but suppress the last blank line
 
-    def inline_tag_to_text(self, tag) -> list:
+    def inline_tag_to_text(self, tag) -> Tuple:
         """Convert html tag to plain text with tag as attributes recursively"""
         markups = self.process_inline_tag_children(tag)
         if not markups:
-            return ""
+            return (tag.name, "")
         return (tag.name, markups)
 
-    def process_inline_tag_children(self, tag) -> list:
+    def process_inline_tag_children(self, tag) -> List:
         """Recursively retrieve all children
         and convert to a list of markup text"""
         markups = []
@@ -133,10 +133,10 @@ def basic_block_tag_handler(self, tag) -> urwid.Widget:
         """default for block tags that need no special treatment"""
         return urwid.Pile(self.process_block_tag_children(tag))
 
-    def _a(self, tag) -> list:
+    def _a(self, tag) -> Tuple:
         markups = self.process_inline_tag_children(tag)
         if not markups:
-            return ""
+            return(tag.name, "")
 
         # hashtag anchors have a class of "mention hashtag"
         # we'll return style "class_mention_hashtag"
@@ -167,7 +167,7 @@ def _blockquote(self, tag) -> urwid.Widget:
         )
         return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")])
 
-    def _br(self, tag) -> list:
+    def _br(self, tag) -> Tuple:
         return (tag.name, ("br", "\n"))
 
     _div = basic_block_tag_handler
@@ -204,11 +204,11 @@ def _pre(self, tag) -> urwid.Widget:
         )
         return urwid.Pile([urwid.AttrMap(pre_widget, "pre")])
 
-    def _span(self, tag) -> list:
+    def _span(self, tag) -> Tuple:
         markups = self.process_inline_tag_children(tag)
 
         if not markups:
-            return ""
+            return (tag.name, "")
 
         # span inherits its parent's class definition
         # unless it has a specific class definition

From 486cd6c7f9c7534d72005aa77562215ff66ad4b7 Mon Sep 17 00:00:00 2001
From: Dan Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 30/55] Make HTML class handling more sensible

If the class name appears in the constants.py PALETTE entry, it is
honored. Otherwise, the class is ignored and the tag is handled
as a generic tag of that type.  This allows hashtag anchors
to be highlighted, and URL anchors to be styled differently
regardless of the strange class markup that Akkoma adds to URL
anchors
---
 toot/tui/constants.py |  5 +----
 toot/tui/richtext.py  | 44 +++++++++++++++++++++++++------------------
 2 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index e2c2fd0a..da5dd657 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -40,10 +40,7 @@
     ('white_bold', 'white,bold', ''),
 
     # HTML tag styling
-    # note, anchor styling is often overridden
-    # by class names in Mastodon statuses
-    # so you won't see the italics.
-    ('a', ',italics', ''),
+    ('a', '', ''),
     ('em', ',italics', ''),
     ('i', ',italics', ''),
     ('strong', ',bold', ''),
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 225d52ee..f1783406 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -4,18 +4,23 @@
 from typing import List, Tuple
 import urwid
 import unicodedata
+from .constants import PALETTE
 from bs4 import BeautifulSoup
 from bs4.element import NavigableString, Tag
 
 
 class ContentParser:
     def __init__(self):
+        self.palette_names = []
+        for p in PALETTE:
+            self.palette_names.append(p[0])
+
         """Parse a limited subset of HTML and create urwid widgets."""
 
     def html_to_widgets(self, html) -> List[urwid.Widget]:
         """Convert html to urwid widgets"""
         widgets: List[urwid.Widget] = []
-        html = unicodedata.normalize('NFKC', html)
+        html = unicodedata.normalize("NFKC", html)
         soup = BeautifulSoup(html.replace("&apos;", "'"), "html.parser")
         for e in soup.body or soup:
             if isinstance(e, NavigableString):
@@ -108,20 +113,18 @@ def get_urwid_attr_name(self, tag) -> str:
         """Get the class name and translate to a
         name suitable for use as an urwid
         text attribute name"""
-        # TODO: think about whitelisting allowed classes,
-        # or blacklisting classes we do not want.
-        # Classes to whitelist: "mention" "hashtag"
-        # used in anchor tags
-        # Classes to blacklist: "invisible" used in Akkoma
-        # anchor titles
 
         if "class" in tag.attrs:
             clss = tag.attrs["class"]
             if len(clss) > 0:
                 style_name = "class_" + "_".join(clss)
-                return style_name
+                # return the class name, only if we
+                # find it as a defined palette name
+                if style_name in self.palette_names:
+                    return style_name
 
-        style_name = tag.name
+        # fallback to returning the tag name
+        return tag.name
 
     # Tag handlers start here.
     # Tags not explicitly listed are "supported" by
@@ -136,12 +139,12 @@ def basic_block_tag_handler(self, tag) -> urwid.Widget:
     def _a(self, tag) -> Tuple:
         markups = self.process_inline_tag_children(tag)
         if not markups:
-            return(tag.name, "")
+            return (tag.name, "")
 
         # hashtag anchors have a class of "mention hashtag"
         # we'll return style "class_mention_hashtag"
-        # in that case; set this up in constants.py
-        # to control highlighting of hashtags
+        # in that case; see corresponding palette entry
+        # in constants.py controlling hashtag highlighting
 
         return (self.get_urwid_attr_name(tag), markups)
 
@@ -216,12 +219,15 @@ def _span(self, tag) -> Tuple:
 
         if "class" in tag.attrs:
             style_name = self.get_urwid_attr_name(tag)
-        elif tag.parent:
-            style_name = self.get_urwid_attr_name(tag.parent)
-        else:
-            style_name = tag.name
+            if style_name != "span":
+                # unique class name matches an entry in our palette
+                return (style_name, markups)
 
-        return (style_name, markups)
+        if tag.parent:
+            return (self.get_urwid_attr_name(tag.parent), markups)
+        else:
+            # fallback
+            return ("span", markups)
 
     def _ul(self, tag) -> urwid.Widget:
         return self.list_widget(tag, ordered=False)
@@ -241,7 +247,9 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
                         ("li", [str(i), ". ", markup])
                     )  # 1. foo, 2. bar, etc.
                 else:
-                    txt = urwid.Text(("li", ["\N{bullet} ", markup]))  # * foo, * bar, etc.
+                    txt = urwid.Text(
+                        ("li", ["\N{bullet} ", markup])
+                    )  # * foo, * bar, etc.
                 widgets.append(txt)
             else:
                 if ordered:

From 103767d677a0e90152cddd10ae62dbe87ebcfa1f Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 31/55] switch pre and code to render as light grey rather than
 dark

---
 toot/tui/constants.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index da5dd657..0a863d2c 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -47,8 +47,8 @@
     ('b', ',bold', ''),
     ('u', ',underline', ''),
     ('del', ',strikethrough', ''),
-    ('code', 'dark gray, standout', ''),
-    ('pre', 'dark gray, standout', ''),
+    ('code', 'light gray, standout', ''),
+    ('pre', 'light gray, standout', ''),
     ('blockquote', 'light gray', ''),
     ('h1', ',bold', ''),
     ('h2', ',bold', ''),

From eb64cb9ca387831bf43d49bf805c40823e5b767c Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 4 Apr 2023 19:43:37 -0400
Subject: [PATCH 32/55] Handle nested B and I tags, also nested EM and STRONG
 tags

---
 toot/tui/constants.py |  6 ++++--
 toot/tui/richtext.py  | 50 ++++++++++++++++++++++++++++++++++++-------
 2 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index 0a863d2c..10897e95 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -41,10 +41,12 @@
 
     # HTML tag styling
     ('a', '', ''),
-    ('em', ',italics', ''),
+    # em tag is mapped to i
     ('i', ',italics', ''),
-    ('strong', ',bold', ''),
+    # strong tag is mapped to b
     ('b', ',bold', ''),
+    # special case for bold + italic nested tags
+    ('bi', ',bold,italics', ''),
     ('u', ',underline', ''),
     ('del', ',strikethrough', ''),
     ('code', 'light gray, standout', ''),
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index f1783406..be00eede 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -173,20 +173,23 @@ def _blockquote(self, tag) -> urwid.Widget:
     def _br(self, tag) -> Tuple:
         return (tag.name, ("br", "\n"))
 
-    _div = basic_block_tag_handler
-
-    _li = basic_block_tag_handler
+    def _em(self, tag) -> Tuple:
+        # to simplify the number of palette entries
+        # translate EM to I (italic)
+        markups = self.process_inline_tag_children(tag)
+        if not markups:
+            return ("i", "")
 
-    # Glitch-soc and Pleroma allow <H1>...<H6> in content
-    # Mastodon (PR #23913) does not; header tags are converted to <P><STRONG></STRONG></P>
+        # special case processing for bold and italic
+        for parent in tag.parents:
+            if parent.name == "b" or parent.name == "strong":
+                return ("bi", markups)
 
-    _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
+        return ("i", markups)
 
     def _ol(self, tag) -> urwid.Widget:
         return self.list_widget(tag, ordered=True)
 
-    _p = basic_block_tag_handler
-
     def _pre(self, tag) -> urwid.Widget:
 
         # <PRE> tag spec says that text should not wrap,
@@ -229,6 +232,20 @@ def _span(self, tag) -> Tuple:
             # fallback
             return ("span", markups)
 
+    def _strong(self, tag) -> Tuple:
+        # to simplify the number of palette entries
+        # translate STRONG to B (bold)
+        markups = self.process_inline_tag_children(tag)
+        if not markups:
+            return ("b", "")
+
+        # special case processing for bold and italic
+        for parent in tag.parents:
+            if parent.name == "i" or parent.name == "em":
+                return ("bi", markups)
+
+        return ("b", markups)
+
     def _ul(self, tag) -> urwid.Widget:
         return self.list_widget(tag, ordered=False)
 
@@ -264,3 +281,20 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
             i += 1
 
         return urwid.Pile(widgets)
+
+    # These tags are handled identically to others
+
+    _b = _strong
+
+    _div = basic_block_tag_handler
+
+    _i = _em
+
+    _li = basic_block_tag_handler
+
+    # Glitch-soc and Pleroma allow <H1>...<H6> in content
+    # Mastodon (PR #23913) does not; header tags are converted to <P><STRONG></STRONG></P>
+
+    _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
+
+    _p = basic_block_tag_handler

From 72aa9341df68cee80e0d618d9024219a7662ee1d Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Wed, 5 Apr 2023 17:14:51 -0400
Subject: [PATCH 33/55] make pytest work properly

---
 .gitignore | 2 --
 Makefile   | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 06bdc4a2..bc647eb2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
 *.egg-info/
 *.pyc
 .pypirc
-.vscode
 /.cache/
 /.coverage
 /.env
@@ -15,4 +14,3 @@
 debug.log
 /pyrightconfig.json
 /book
-/venv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index e46a7399..c413d73a 100644
--- a/Makefile
+++ b/Makefile
@@ -8,9 +8,9 @@ publish :
 	twine upload dist/*.tar.gz dist/*.whl
 
 test:
-	pytest -v
+	pytest tests/*.py -v
 	flake8
-	vermin --target=3.6 --no-tips --violations --exclude-regex 'venv/.*' .
+	vermin --target=3.6 --no-tips --violations --exclude-regex venv/.* .
 
 coverage:
 	coverage erase

From e9b4bfb5d58053ee0b3c3b4feb99c44e978caaa6 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 6 Apr 2023 10:00:23 -0400
Subject: [PATCH 34/55] fix markup for br tag

---
 toot/tui/richtext.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index be00eede..97586b40 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -171,7 +171,7 @@ def _blockquote(self, tag) -> urwid.Widget:
         return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")])
 
     def _br(self, tag) -> Tuple:
-        return (tag.name, ("br", "\n"))
+        return ("br", "\n")
 
     def _em(self, tag) -> Tuple:
         # to simplify the number of palette entries

From bb31054fd4e32b609bd508b4a7c34725b131f35d Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 6 Apr 2023 23:24:44 -0400
Subject: [PATCH 35/55] elaborated comment

---
 toot/tui/richtext.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 97586b40..6d83804a 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -283,6 +283,8 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
         return urwid.Pile(widgets)
 
     # These tags are handled identically to others
+    # the only difference being the tag name used for
+    # urwid attribute mapping
 
     _b = _strong
 

From 306cf45861d20da71a2d606c5d641368f22b1cc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADtor=20Galv=C3=A3o?= <info@vitorgalvao.com>
Date: Tue, 18 Apr 2023 19:53:55 +0100
Subject: [PATCH 36/55] README.rst: Fix image links

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 90751b9b..cd6c572e 100644
--- a/README.rst
+++ b/README.rst
@@ -39,9 +39,9 @@ Terminal User Interface
 
 toot includes a terminal user interface (TUI). Run it with ``toot tui``.
 
-.. image :: https://raw.githubusercontent.com/ihabunek/toot/master/docs/_static/tui_list.png
+.. image :: https://raw.githubusercontent.com/ihabunek/toot/master/docs/images/tui_list.png
 
-.. image :: https://raw.githubusercontent.com/ihabunek/toot/master/docs/_static/tui_compose.png
+.. image :: https://raw.githubusercontent.com/ihabunek/toot/master/docs/images/tui_compose.png
 
 
 License

From 5d47a0ad76e806a3f5d1df5f96ec8230d1ee09a9 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Mon, 1 May 2023 09:24:20 -0400
Subject: [PATCH 37/55] added urwidgets dependency

---
 .vscode/launch.json  | 24 ++++++++++++++++++++++++
 requirements-dev.txt |  1 +
 requirements.txt     |  2 +-
 setup.py             |  1 +
 toot/console.py      |  4 ++++
 toot/debugger.py     | 10 ++++++++++
 6 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 .vscode/launch.json
 create mode 100644 toot/debugger.py

diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 00000000..93f37d2e
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,24 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Remote Attach",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 9000
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "."
+                }
+            ],
+            "justMyCode": false
+        }
+    ]
+}
\ No newline at end of file
diff --git a/requirements-dev.txt b/requirements-dev.txt
index dfa5b15a..e3976a53 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -6,3 +6,4 @@ sphinx
 sphinx-autobuild
 twine
 wheel
+debugpy
diff --git a/requirements.txt b/requirements.txt
index 3616ac32..014fedac 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ requests>=2.13,<3.0
 beautifulsoup4>=4.5.0,<5.0
 wcwidth>=0.1.7
 urwid>=2.0.0,<3.0
-
+urwidgets=1.0.0.dev0
diff --git a/setup.py b/setup.py
index 07c2b63a..9365b64b 100644
--- a/setup.py
+++ b/setup.py
@@ -38,6 +38,7 @@
         "beautifulsoup4>=4.5.0,<5.0",
         "wcwidth>=0.1.7",
         "urwid>=2.0.0,<3.0",
+        "urwidgets>=1.0.0.dev0",
     ],
     entry_points={
         'console_scripts': [
diff --git a/toot/console.py b/toot/console.py
index 5c84d97c..3a753965 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -10,6 +10,7 @@
 from toot import config, commands, CLIENT_NAME, CLIENT_WEBSITE, __version__
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import print_out, print_err
+from .debugger import initialize_debugger
 
 VISIBILITY_CHOICES = ["public", "unlisted", "private", "direct"]
 VISIBILITY_CHOICES_STR = ", ".join(f"'{v}'" for v in VISIBILITY_CHOICES)
@@ -901,6 +902,9 @@ def run_command(app, user, name, args):
 
 
 def main():
+    if "--debugger" in sys.argv:
+        initialize_debugger()
+
     # Enable debug logging if --debug is in args
     if "--debug" in sys.argv:
         filename = os.getenv("TOOT_LOG_FILE")
diff --git a/toot/debugger.py b/toot/debugger.py
new file mode 100644
index 00000000..bc85d829
--- /dev/null
+++ b/toot/debugger.py
@@ -0,0 +1,10 @@
+def initialize_debugger():
+    import multiprocessing
+
+    if multiprocessing.current_process().pid > 1:
+        import debugpy
+
+        debugpy.listen(("0.0.0.0", 9000))
+        print("VSCode Debugger is ready to be attached on port 9000, press F5", flush=True)
+        debugpy.wait_for_client()
+        print("VSCode Debugger is now attached", flush=True)

From 9ccfd6b4e27ded75b41284a9039991cef4c0f9a0 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Sun, 7 May 2023 23:05:58 -0400
Subject: [PATCH 38/55] Support OSC 8 underline of anchor links

Uses the Hyperlink widget along with TextEmbed widget
---
 toot/tui/richtext.py | 52 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 40 insertions(+), 12 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 6d83804a..45b4cd8a 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -2,11 +2,14 @@
 richtext
 """
 from typing import List, Tuple
+import re
 import urwid
 import unicodedata
 from .constants import PALETTE
 from bs4 import BeautifulSoup
 from bs4.element import NavigableString, Tag
+from urwidgets import TextEmbed, Hyperlink, parse_text
+from urwid.util import decompose_tagmarkup
 
 
 class ContentParser:
@@ -33,7 +36,7 @@ def html_to_widgets(self, html) -> List[urwid.Widget]:
 
             if not isinstance(markup, urwid.Widget):
                 # plaintext, so create a padded text widget
-                txt = urwid.Text(markup)
+                txt = self.text_to_widget("", markup)
                 markup = urwid.Padding(
                     txt,
                     align="left",
@@ -65,6 +68,28 @@ def process_inline_tag_children(self, tag) -> List:
                 markups.append(child)
         return markups
 
+    def text_to_widget(self, attr, markup) -> TextEmbed:
+        TRANSFORM = {
+            # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
+            re.compile(r'(^.+)~~~(.+$)'):
+                lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[2], attr[0], g[1]))),
+        }
+        markup_list = []
+
+        for run in markup:
+            if isinstance(run, tuple):
+                txt, attr = decompose_tagmarkup(run)
+                m = re.match(r'(^.+)~~~(.+$)', txt)
+                if m:
+                    markup_list.append(parse_text(txt, TRANSFORM,
+                    lambda pattern, groups, span: TRANSFORM[pattern](groups)))
+                else:
+                    markup_list.append(run)
+            else:
+                markup_list.append(run)
+
+        return TextEmbed(markup_list)
+
     def process_block_tag_children(self, tag) -> List[urwid.Widget]:
         """Recursively retrieve all children
         and convert to a list of widgets
@@ -99,13 +124,13 @@ def process_block_tag_children(self, tag) -> List[urwid.Widget]:
 
         widget_list = []
         if len(pre_widget_markups):
-            widget_list.append(urwid.Text((tag.name, pre_widget_markups)))
+            widget_list.append(self.text_to_widget(tag.name, pre_widget_markups))
 
         if len(child_widgets):
             widget_list += child_widgets
 
         if len(post_widget_markups):
-            widget_list.append(urwid.Text((tag.name, post_widget_markups)))
+            widget_list.append(self.text_to_widget(tag.name, post_widget_markups))
 
         return widget_list
 
@@ -141,12 +166,17 @@ def _a(self, tag) -> Tuple:
         if not markups:
             return (tag.name, "")
 
+        href = tag.attrs["href"]
+        title, title_attrib = decompose_tagmarkup(markups)
+        if href:
+            title += f"~~~{href}"
+
         # hashtag anchors have a class of "mention hashtag"
         # we'll return style "class_mention_hashtag"
         # in that case; see corresponding palette entry
         # in constants.py controlling hashtag highlighting
 
-        return (self.get_urwid_attr_name(tag), markups)
+        return (self.get_urwid_attr_name(tag), title)
 
     def _blockquote(self, tag) -> urwid.Widget:
         widget_list = self.process_block_tag_children(tag)
@@ -260,19 +290,17 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
 
             if not isinstance(markup, urwid.Widget):
                 if ordered:
-                    txt = urwid.Text(
-                        ("li", [str(i), ". ", markup])
-                    )  # 1. foo, 2. bar, etc.
+                    txt = self.text_to_widget("li", [str(i), ". ", markup])
+                    # 1. foo, 2. bar, etc.
                 else:
-                    txt = urwid.Text(
-                        ("li", ["\N{bullet} ", markup])
-                    )  # * foo, * bar, etc.
+                    txt = self.text_to_widget("li", ["\N{bullet} ", markup])
+                    # * foo, * bar, etc.
                 widgets.append(txt)
             else:
                 if ordered:
-                    txt = urwid.Text(("li", [str(i) + "."]))
+                    txt = self.text_to_widget("li", [str(i), ". "])
                 else:
-                    txt = urwid.Text(("li", "\N{bullet}"))
+                    txt = self.text_to_widget("li", ["\N{bullet} "])
 
                 columns = urwid.Columns(
                     [txt, ("weight", 9999, markup)], dividechars=1, min_width=3

From 4a5db5bed840aacdf0b123204d577486fe4cd55c Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Mon, 8 May 2023 00:06:01 -0400
Subject: [PATCH 39/55] added OSC 8 hyperlinks for media links and card links

---
 toot/tui/richtext.py |  8 +++++---
 toot/tui/timeline.py | 18 ++++++++++++++++--
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 45b4cd8a..26d9dbc7 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -71,7 +71,7 @@ def process_inline_tag_children(self, tag) -> List:
     def text_to_widget(self, attr, markup) -> TextEmbed:
         TRANSFORM = {
             # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
-            re.compile(r'(^.+)~~~(.+$)'):
+            re.compile(r'(^.+)\x03(.+$)'):
                 lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[2], attr[0], g[1]))),
         }
         markup_list = []
@@ -79,7 +79,7 @@ def text_to_widget(self, attr, markup) -> TextEmbed:
         for run in markup:
             if isinstance(run, tuple):
                 txt, attr = decompose_tagmarkup(run)
-                m = re.match(r'(^.+)~~~(.+$)', txt)
+                m = re.match(r'(^.+)\x03(.+$)', txt)
                 if m:
                     markup_list.append(parse_text(txt, TRANSFORM,
                     lambda pattern, groups, span: TRANSFORM[pattern](groups)))
@@ -169,7 +169,9 @@ def _a(self, tag) -> Tuple:
         href = tag.attrs["href"]
         title, title_attrib = decompose_tagmarkup(markups)
         if href:
-            title += f"~~~{href}"
+            # use \x03 (ETX) as a sentinel character
+            # to signal a href following
+            title += f"\x03{href}"
 
         # hashtag anchors have a class of "mention hashtag"
         # we'll return style "class_mention_hashtag"
diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py
index b102e294..3fb9b350 100644
--- a/toot/tui/timeline.py
+++ b/toot/tui/timeline.py
@@ -1,4 +1,5 @@
 import logging
+import re
 import sys
 import urwid
 import webbrowser
@@ -13,6 +14,7 @@
 from toot.tui import app
 from toot.tui.utils import time_ago
 from toot.utils.language import language_name
+from urwidgets import Hyperlink, TextEmbed, parse_text
 
 logger = logging.getLogger("toot")
 
@@ -319,6 +321,18 @@ def __init__(self, timeline: Timeline, status: Optional[Status]):
             if status else ())
         return super().__init__(widget_list)
 
+    def linkify_content(self, text) -> urwid.Widget:
+        TRANSFORM = {
+            # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
+            re.compile(r'(https?://[^\s]+)'):
+                lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[1], "link"))),
+        }
+        markup_list = []
+
+        markup_list.append(parse_text(text, TRANSFORM,
+            lambda pattern, groups, span: TRANSFORM[pattern](groups)))
+        return TextEmbed(markup_list, align='left')
+
     def content_generator(self, status, reblogged_by):
         if reblogged_by:
             text = "♺ {} boosted".format(reblogged_by.display_name or reblogged_by.username)
@@ -355,7 +369,7 @@ def content_generator(self, status, reblogged_by):
                     yield ("pack", urwid.Text([("bold", "Media attachment"), " (", m["type"], ")"]))
                     if m["description"]:
                         yield ("pack", urwid.Text(m["description"]))
-                    yield ("pack", urwid.Text(("link", m["url"])))
+                    yield ("pack", self.linkify_content(m["url"]))
 
             poll = status.original.data.get("poll")
             if poll:
@@ -415,7 +429,7 @@ def card_generator(self, card):
         if card["description"]:
             yield urwid.Text(card["description"].strip())
             yield urwid.Text("")
-        yield urwid.Text(("link", card["url"]))
+        yield self.linkify_content(card["url"])
 
     def poll_generator(self, poll):
         for idx, option in enumerate(poll["options"]):

From baa50ca8893f0de295601dc33e11662e57faba60 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Fri, 12 May 2023 20:24:16 -0400
Subject: [PATCH 40/55] Added a workaround for statuses with malformed HTML

We see this problem with statuses from Pixelfed servers.
Per the Mastodon API spec, the content tag is supposed to be
HTML, but Pixelfed sends statuses that often start as plain text.
They may include embedded anchor tags etc. within the text.
This confuses BeautifulSoup HTML parsers and results in bad
rendering artifacts.

This workaround detects the above condition and attempts to fix it by
surrounding the status in <p></p>. This converts it to nominally
valid HTML (at least, parseable by BeautifulSoup.)
---
 toot/tui/richtext.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 26d9dbc7..45a5e342 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -20,19 +20,29 @@ def __init__(self):
 
         """Parse a limited subset of HTML and create urwid widgets."""
 
-    def html_to_widgets(self, html) -> List[urwid.Widget]:
+    def html_to_widgets(self, html, recovery_attempt = False) -> List[urwid.Widget]:
         """Convert html to urwid widgets"""
         widgets: List[urwid.Widget] = []
         html = unicodedata.normalize("NFKC", html)
         soup = BeautifulSoup(html.replace("&apos;", "'"), "html.parser")
+        first_tag = True
         for e in soup.body or soup:
             if isinstance(e, NavigableString):
-                continue
-            name = e.name
-            # First, look for a custom tag handler method in this class
-            # If that fails, fall back to inline_tag_to_text handler
-            method = getattr(self, "_" + name, self.inline_tag_to_text)
-            markup = method(e)  # either returns a Widget, or plain text
+                if first_tag and not recovery_attempt:
+                    # if our first "tag" is a navigable string
+                    # the HTML is out of spec, doesn't start with a tag,
+                    # we see this in content from Pixelfed servers.
+                    # attempt a fix by wrapping the HTML with <p></p>
+                    return self.html_to_widgets(f"<p>{html}</p>", recovery_attempt = True)
+                else:
+                    continue
+            else:
+                first_tag = False
+                name = e.name
+                # First, look for a custom tag handler method in this class
+                # If that fails, fall back to inline_tag_to_text handler
+                method = getattr(self, "_" + name, self.inline_tag_to_text)
+                markup = method(e)  # either returns a Widget, or plain text
 
             if not isinstance(markup, urwid.Widget):
                 # plaintext, so create a padded text widget
@@ -72,13 +82,13 @@ def text_to_widget(self, attr, markup) -> TextEmbed:
         TRANSFORM = {
             # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
             re.compile(r'(^.+)\x03(.+$)'):
-                lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[2], attr[0], g[1]))),
+                lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[2], attr, g[1]))),
         }
         markup_list = []
 
         for run in markup:
             if isinstance(run, tuple):
-                txt, attr = decompose_tagmarkup(run)
+                txt, attr_list = decompose_tagmarkup(run)
                 m = re.match(r'(^.+)\x03(.+$)', txt)
                 if m:
                     markup_list.append(parse_text(txt, TRANSFORM,

From de3461e19caac53cb3bbc74b78c269e4b86011a4 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Fri, 12 May 2023 20:24:16 -0400
Subject: [PATCH 41/55] Hashtag highlighting now works correctly

Also, hashtags are created as OCS-8 hyperlinks, so they are
directly clickable in terminals that support OCS-8
---
 toot/tui/constants.py |  4 +--
 toot/tui/richtext.py  | 61 ++++++++++++++++++++++++++++++++++---------
 2 files changed, 51 insertions(+), 14 deletions(-)

diff --git a/toot/tui/constants.py b/toot/tui/constants.py
index 10897e95..0039de87 100644
--- a/toot/tui/constants.py
+++ b/toot/tui/constants.py
@@ -40,7 +40,7 @@
     ('white_bold', 'white,bold', ''),
 
     # HTML tag styling
-    ('a', '', ''),
+    ('a', ',italics', ''),
     # em tag is mapped to i
     ('i', ',italics', ''),
     # strong tag is mapped to b
@@ -59,7 +59,7 @@
     ('h5', ',bold', ''),
     ('h6', ',bold', ''),
     ('class_mention_hashtag', 'light cyan,bold', ''),
-
+    ('class_hashtag', 'light cyan,bold', ''),
 ]
 
 VISIBILITY_OPTIONS = [
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 45a5e342..607a46c8 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -20,7 +20,7 @@ def __init__(self):
 
         """Parse a limited subset of HTML and create urwid widgets."""
 
-    def html_to_widgets(self, html, recovery_attempt = False) -> List[urwid.Widget]:
+    def html_to_widgets(self, html, recovery_attempt=False) -> List[urwid.Widget]:
         """Convert html to urwid widgets"""
         widgets: List[urwid.Widget] = []
         html = unicodedata.normalize("NFKC", html)
@@ -33,7 +33,7 @@ def html_to_widgets(self, html, recovery_attempt = False) -> List[urwid.Widget]:
                     # the HTML is out of spec, doesn't start with a tag,
                     # we see this in content from Pixelfed servers.
                     # attempt a fix by wrapping the HTML with <p></p>
-                    return self.html_to_widgets(f"<p>{html}</p>", recovery_attempt = True)
+                    return self.html_to_widgets(f"<p>{html}</p>", recovery_attempt=True)
                 else:
                     continue
             else:
@@ -81,18 +81,26 @@ def process_inline_tag_children(self, tag) -> List:
     def text_to_widget(self, attr, markup) -> TextEmbed:
         TRANSFORM = {
             # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
-            re.compile(r'(^.+)\x03(.+$)'):
-                lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[2], attr, g[1]))),
+            re.compile(r"(^.+)\x03(.+$)"): lambda g: (
+                len(g[1]),
+                urwid.Filler(Hyperlink(g[2], anchor_attr, g[1])),
+            ),
         }
         markup_list = []
 
         for run in markup:
             if isinstance(run, tuple):
                 txt, attr_list = decompose_tagmarkup(run)
-                m = re.match(r'(^.+)\x03(.+$)', txt)
+                m = re.match(r"(^.+)\x03(.+$)", txt)
                 if m:
-                    markup_list.append(parse_text(txt, TRANSFORM,
-                    lambda pattern, groups, span: TRANSFORM[pattern](groups)))
+                    anchor_attr = self.get_best_anchor_attr(attr_list)
+                    markup_list.append(
+                        parse_text(
+                            txt,
+                            TRANSFORM,
+                            lambda pattern, groups, span: TRANSFORM[pattern](groups),
+                        )
+                    )
                 else:
                     markup_list.append(run)
             else:
@@ -171,24 +179,46 @@ def basic_block_tag_handler(self, tag) -> urwid.Widget:
         """default for block tags that need no special treatment"""
         return urwid.Pile(self.process_block_tag_children(tag))
 
+    def get_best_anchor_attr(self, attrib_list) -> str:
+        if not attrib_list:
+            return ""
+        flat_al = list(flatten(attrib_list))
+
+        for a in flat_al[0]:
+            # ref: https://docs.joinmastodon.org/spec/activitypub/
+            # these are the class names (translated to attrib names)
+            # that we can support for display
+
+            try:
+                if a[0] in ["class_hashtag", "class_mention_hashtag", "class_mention"]:
+                    return a[0]
+            except KeyError:
+                continue
+
+        return "a"
+
     def _a(self, tag) -> Tuple:
         markups = self.process_inline_tag_children(tag)
         if not markups:
             return (tag.name, "")
 
         href = tag.attrs["href"]
-        title, title_attrib = decompose_tagmarkup(markups)
+        title, attrib_list = decompose_tagmarkup(markups)
+        if not attrib_list:
+            attrib_list = [tag]
         if href:
-            # use \x03 (ETX) as a sentinel character
-            # to signal a href following
             title += f"\x03{href}"
 
+        attr = self.get_best_anchor_attr(attrib_list)
+
         # hashtag anchors have a class of "mention hashtag"
+        # or "hashtag"
         # we'll return style "class_mention_hashtag"
+        # or "class_hashtag"
         # in that case; see corresponding palette entry
         # in constants.py controlling hashtag highlighting
 
-        return (self.get_urwid_attr_name(tag), title)
+        return (attr, title)
 
     def _blockquote(self, tag) -> urwid.Widget:
         widget_list = self.process_block_tag_children(tag)
@@ -233,7 +263,6 @@ def _ol(self, tag) -> urwid.Widget:
         return self.list_widget(tag, ordered=True)
 
     def _pre(self, tag) -> urwid.Widget:
-
         # <PRE> tag spec says that text should not wrap,
         # but horizontal screen space is at a premium
         # and we have no horizontal scroll bar, so allow
@@ -340,3 +369,11 @@ def list_widget(self, tag, ordered=False) -> urwid.Widget:
     _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
 
     _p = basic_block_tag_handler
+
+
+def flatten(data):
+    if isinstance(data, tuple):
+        for x in data:
+            yield from flatten(x)
+    else:
+        yield data

From 457facde885ad6698521f5f735872004f3f3eb65 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Fri, 12 May 2023 20:24:16 -0400
Subject: [PATCH 42/55] Fix for rendering HTML that starts with an inline tag

Also, now renders HTML in Account overlay page
---
 .vscode/launch.json  | 24 ------------------------
 requirements-dev.txt |  1 -
 toot/console.py      |  4 ----
 toot/debugger.py     | 10 ----------
 toot/tui/overlays.py | 21 ++++++++++++++-------
 toot/tui/richtext.py | 24 +++++++++++++++++++++++-
 6 files changed, 37 insertions(+), 47 deletions(-)
 delete mode 100644 .vscode/launch.json
 delete mode 100644 toot/debugger.py

diff --git a/.vscode/launch.json b/.vscode/launch.json
deleted file mode 100644
index 93f37d2e..00000000
--- a/.vscode/launch.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Python: Remote Attach",
-            "type": "python",
-            "request": "attach",
-            "connect": {
-                "host": "localhost",
-                "port": 9000
-            },
-            "pathMappings": [
-                {
-                    "localRoot": "${workspaceFolder}",
-                    "remoteRoot": "."
-                }
-            ],
-            "justMyCode": false
-        }
-    ]
-}
\ No newline at end of file
diff --git a/requirements-dev.txt b/requirements-dev.txt
index e3976a53..dfa5b15a 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -6,4 +6,3 @@ sphinx
 sphinx-autobuild
 twine
 wheel
-debugpy
diff --git a/toot/console.py b/toot/console.py
index 3a753965..5c84d97c 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -10,7 +10,6 @@
 from toot import config, commands, CLIENT_NAME, CLIENT_WEBSITE, __version__
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import print_out, print_err
-from .debugger import initialize_debugger
 
 VISIBILITY_CHOICES = ["public", "unlisted", "private", "direct"]
 VISIBILITY_CHOICES_STR = ", ".join(f"'{v}'" for v in VISIBILITY_CHOICES)
@@ -902,9 +901,6 @@ def run_command(app, user, name, args):
 
 
 def main():
-    if "--debugger" in sys.argv:
-        initialize_debugger()
-
     # Enable debug logging if --debug is in args
     if "--debug" in sys.argv:
         filename = os.getenv("TOOT_LOG_FILE")
diff --git a/toot/debugger.py b/toot/debugger.py
deleted file mode 100644
index bc85d829..00000000
--- a/toot/debugger.py
+++ /dev/null
@@ -1,10 +0,0 @@
-def initialize_debugger():
-    import multiprocessing
-
-    if multiprocessing.current_process().pid > 1:
-        import debugpy
-
-        debugpy.listen(("0.0.0.0", 9000))
-        print("VSCode Debugger is ready to be attached on port 9000, press F5", flush=True)
-        debugpy.wait_for_client()
-        print("VSCode Debugger is now attached", flush=True)
diff --git a/toot/tui/overlays.py b/toot/tui/overlays.py
index a9c24428..83b19676 100644
--- a/toot/tui/overlays.py
+++ b/toot/tui/overlays.py
@@ -4,10 +4,10 @@
 import webbrowser
 
 from toot import __version__
-from toot.utils import format_content
-from .utils import highlight_hashtags, highlight_keys
-from .widgets import Button, EditBox, SelectableText
 from toot import api
+from .utils import highlight_keys
+from .widgets import Button, EditBox, SelectableText
+from .richtext import ContentParser
 
 
 class StatusSource(urwid.Padding):
@@ -250,6 +250,8 @@ def setup_listbox(self):
         super().__init__(walker)
 
     def generate_contents(self, account, relationship=None, last_action=None):
+        parser = ContentParser()
+
         if self.last_action and not self.last_action.startswith("Confirm"):
             yield Button(f"Confirm {self.last_action}", on_press=take_action, user_data=self)
             yield Button("Cancel", on_press=cancel_action, user_data=self)
@@ -274,8 +276,10 @@ def generate_contents(self, account, relationship=None, last_action=None):
 
         if account["note"]:
             yield urwid.Divider()
-            for line in format_content(account["note"]):
-                yield urwid.Text(highlight_hashtags(line, followed_tags=set()))
+
+            widgetlist = parser.html_to_widgets(account["note"])
+            for line in widgetlist:
+                yield (line)
 
         yield urwid.Divider()
         yield urwid.Text(["ID: ", ("green", f"{account['id']}")])
@@ -307,8 +311,11 @@ def generate_contents(self, account, relationship=None, last_action=None):
                 name = field["name"].title()
                 yield urwid.Divider()
                 yield urwid.Text([("yellow", f"{name.rstrip(':')}"), ":"])
-                for line in format_content(field["value"]):
-                    yield urwid.Text(highlight_hashtags(line, followed_tags=set()))
+
+                widgetlist = parser.html_to_widgets(field["value"])
+                for line in widgetlist:
+                    yield (line)
+
                 if field["verified_at"]:
                     yield urwid.Text(("green", "✓ Verified"))
 
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 607a46c8..c188f78a 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -37,12 +37,34 @@ def html_to_widgets(self, html, recovery_attempt=False) -> List[urwid.Widget]:
                 else:
                     continue
             else:
-                first_tag = False
                 name = e.name
+                # if our HTML starts with a tag, but not a block tag
+                # the HTML is out of spec. Attempt a fix by wrapping the
+                # HTML with <p></p>
+                if (
+                    first_tag
+                    and not recovery_attempt
+                    and name
+                    not in (
+                        "p",
+                        "pre",
+                        "li",
+                        "blockquote",
+                        "h1",
+                        "h2",
+                        "h3",
+                        "h4",
+                        "h5",
+                        "h6",
+                    )
+                ):
+                    return self.html_to_widgets(f"<p>{html}</p>", recovery_attempt=True)
+
                 # First, look for a custom tag handler method in this class
                 # If that fails, fall back to inline_tag_to_text handler
                 method = getattr(self, "_" + name, self.inline_tag_to_text)
                 markup = method(e)  # either returns a Widget, or plain text
+                first_tag = False
 
             if not isinstance(markup, urwid.Widget):
                 # plaintext, so create a padded text widget

From fe88a40042dd8617ce58376eaa7b1598f5ab0e15 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Mon, 15 May 2023 18:29:32 -0400
Subject: [PATCH 43/55] Switched to released version of urwidgets (0.1.0)

Also, removed Python 3.6 tests as urwidgets is Python 3.7+
---
 .github/workflows/test.yml | 5 +++--
 requirements.txt           | 2 +-
 setup.py                   | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e93c9bc6..cf604f9d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,10 +6,11 @@ jobs:
   test:
     # Older Ubuntu required for testing on Python 3.6 which is not available in
     # later versions. Remove once support for 3.6 is dropped.
-    runs-on: ubuntu-20.04
+    # runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3
diff --git a/requirements.txt b/requirements.txt
index 014fedac..54b69dd3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ requests>=2.13,<3.0
 beautifulsoup4>=4.5.0,<5.0
 wcwidth>=0.1.7
 urwid>=2.0.0,<3.0
-urwidgets=1.0.0.dev0
+urwidgets=>=0.1,<0.2
diff --git a/setup.py b/setup.py
index 9365b64b..5abd1a44 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@
         "beautifulsoup4>=4.5.0,<5.0",
         "wcwidth>=0.1.7",
         "urwid>=2.0.0,<3.0",
-        "urwidgets>=1.0.0.dev0",
+        "urwidgets>=0.1,<0.2",
     ],
     entry_points={
         'console_scripts': [

From b3c654b64fa930fe36612ca0a1339f16483074bf Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 16 May 2023 18:48:03 -0400
Subject: [PATCH 44/55] Added support for li value attribute, ol start and
 reverse attributes

---
 toot/tui/richtext.py | 80 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 19 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index c188f78a..abf060ee 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -56,7 +56,7 @@ def html_to_widgets(self, html, recovery_attempt=False) -> List[urwid.Widget]:
                         "h4",
                         "h5",
                         "h6",
-                    )
+                    )  # NOTE: update this list if Mastodon starts supporting more block tags
                 ):
                     return self.html_to_widgets(f"<p>{html}</p>", recovery_attempt=True)
 
@@ -113,6 +113,7 @@ def text_to_widget(self, attr, markup) -> TextEmbed:
         for run in markup:
             if isinstance(run, tuple):
                 txt, attr_list = decompose_tagmarkup(run)
+                # find anchor titles with an ETX separator followed by href
                 m = re.match(r"(^.+)\x03(.+$)", txt)
                 if m:
                     anchor_attr = self.get_best_anchor_attr(attr_list)
@@ -220,6 +221,8 @@ def get_best_anchor_attr(self, attrib_list) -> str:
         return "a"
 
     def _a(self, tag) -> Tuple:
+        """anchor tag handler"""
+
         markups = self.process_inline_tag_children(tag)
         if not markups:
             return (tag.name, "")
@@ -229,6 +232,8 @@ def _a(self, tag) -> Tuple:
         if not attrib_list:
             attrib_list = [tag]
         if href:
+            # use ASCII ETX (end of record) as a
+            # delimiter between the title and the HREF
             title += f"\x03{href}"
 
         attr = self.get_best_anchor_attr(attrib_list)
@@ -282,7 +287,46 @@ def _em(self, tag) -> Tuple:
         return ("i", markups)
 
     def _ol(self, tag) -> urwid.Widget:
-        return self.list_widget(tag, ordered=True)
+        """ordered list tag handler"""
+
+        widgets = []
+        list_item_num = 1
+        increment = -1 if tag.has_attr("reversed") else 1
+
+        # get ol start= attribute if present
+        if tag.has_attr("start") and len(tag.attrs["start"]) > 0:
+            try:
+                list_item_num = int(tag.attrs["start"])
+            except ValueError:
+                pass
+
+        for li in tag.find_all("li", recursive=False):
+            method = getattr(self, "_li", self.inline_tag_to_text)
+            markup = method(li)
+
+            # li value= attribute will change the item number
+            # it also overrides any ol start= attribute
+
+            if li.has_attr("value") and len(li.attrs["value"]) > 0:
+                try:
+                    list_item_num = int(li.attrs["value"])
+                except ValueError:
+                    pass
+
+            if not isinstance(markup, urwid.Widget):
+                txt = self.text_to_widget("li", [str(list_item_num), ". ", markup])
+                # 1. foo, 2. bar, etc.
+                widgets.append(txt)
+            else:
+                txt = self.text_to_widget("li", [str(list_item_num), ". "])
+                columns = urwid.Columns(
+                    [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
+                )
+                widgets.append(columns)
+
+            list_item_num += increment
+
+        return urwid.Pile(widgets)
 
     def _pre(self, tag) -> urwid.Widget:
         # <PRE> tag spec says that text should not wrap,
@@ -314,7 +358,17 @@ def _span(self, tag) -> Tuple:
         # of its own
 
         if "class" in tag.attrs:
+            # uncomment the following code to hide all HTML marked
+            # invisible (generally, the http:// prefix of URLs)
+            # could be a user preference, it's only advisable if
+            # the terminal supports OCS 8 hyperlinks (and that's not
+            # automatically detectable)
+
+            #            if "invisible" in tag.attrs["class"]:
+            #                return (tag.name, "")
+
             style_name = self.get_urwid_attr_name(tag)
+
             if style_name != "span":
                 # unique class name matches an entry in our palette
                 return (style_name, markups)
@@ -340,36 +394,24 @@ def _strong(self, tag) -> Tuple:
         return ("b", markups)
 
     def _ul(self, tag) -> urwid.Widget:
-        return self.list_widget(tag, ordered=False)
+        """unordered list tag handler"""
 
-    def list_widget(self, tag, ordered=False) -> urwid.Widget:
-        """common logic for ordered and unordered list rendering
-        as urwid widgets"""
         widgets = []
-        i = 1
+
         for li in tag.find_all("li", recursive=False):
             method = getattr(self, "_li", self.inline_tag_to_text)
             markup = method(li)
 
             if not isinstance(markup, urwid.Widget):
-                if ordered:
-                    txt = self.text_to_widget("li", [str(i), ". ", markup])
-                    # 1. foo, 2. bar, etc.
-                else:
-                    txt = self.text_to_widget("li", ["\N{bullet} ", markup])
-                    # * foo, * bar, etc.
+                txt = self.text_to_widget("li", ["\N{bullet} ", markup])
+                # * foo, * bar, etc.
                 widgets.append(txt)
             else:
-                if ordered:
-                    txt = self.text_to_widget("li", [str(i), ". "])
-                else:
-                    txt = self.text_to_widget("li", ["\N{bullet} "])
-
+                txt = self.text_to_widget("li", ["\N{bullet} "])
                 columns = urwid.Columns(
                     [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
                 )
                 widgets.append(columns)
-            i += 1
 
         return urwid.Pile(widgets)
 

From 86304dfef098feb777da8d30f55f480c42299a9b Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 16 May 2023 19:51:05 -0400
Subject: [PATCH 45/55] fix ws

---
 toot/tui/richtext.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index abf060ee..fc178be1 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -364,8 +364,8 @@ def _span(self, tag) -> Tuple:
             # the terminal supports OCS 8 hyperlinks (and that's not
             # automatically detectable)
 
-            #            if "invisible" in tag.attrs["class"]:
-            #                return (tag.name, "")
+            # if "invisible" in tag.attrs["class"]:
+            #     return (tag.name, "")
 
             style_name = self.get_urwid_attr_name(tag)
 

From 61092574c5145fe223ca3c71a730a36b0f7329ee Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Tue, 16 May 2023 20:05:20 -0400
Subject: [PATCH 46/55] Removed followed_tags highlight feature (incompatible
 with HTML render)

Also converted polls.py to use HTML rendering
---
 toot/tui/app.py      | 17 -----------------
 toot/tui/poll.py     | 12 ++++++++----
 toot/tui/timeline.py |  1 -
 toot/tui/utils.py    | 15 ---------------
 4 files changed, 8 insertions(+), 37 deletions(-)

diff --git a/toot/tui/app.py b/toot/tui/app.py
index b49dee23..6dfeb306 100644
--- a/toot/tui/app.py
+++ b/toot/tui/app.py
@@ -120,7 +120,6 @@ def __init__(self, app, user, args):
     def run(self):
         self.loop.set_alarm_in(0, lambda *args: self.async_load_instance())
         self.loop.set_alarm_in(0, lambda *args: self.async_load_followed_accounts())
-        self.loop.set_alarm_in(0, lambda *args: self.async_load_followed_tags())
         self.loop.set_alarm_in(0, lambda *args: self.async_load_timeline(
             is_initial=True, timeline_name="home"))
         self.loop.run()
@@ -316,22 +315,6 @@ def _done_accounts(accounts):
 
         self.run_in_thread(_load_accounts, done_callback=_done_accounts)
 
-    def async_load_followed_tags(self):
-        def _load_tag_list():
-            try:
-                return api.followed_tags(self.app, self.user)
-            except ApiError:
-                # not supported by all Mastodon servers so fail silently if necessary
-                return []
-
-        def _done_tag_list(tags):
-            if len(tags) > 0:
-                self.followed_tags = [t["name"] for t in tags]
-            else:
-                self.followed_tags = []
-
-        self.run_in_thread(_load_tag_list, done_callback=_done_tag_list)
-
     def refresh_footer(self, timeline):
         """Show status details in footer."""
         status, index, count = timeline.get_focused_status_with_counts()
diff --git a/toot/tui/poll.py b/toot/tui/poll.py
index 81756af2..8ad649c1 100644
--- a/toot/tui/poll.py
+++ b/toot/tui/poll.py
@@ -2,9 +2,9 @@
 
 from toot import api
 from toot.exceptions import ApiError
-from toot.utils import format_content
-from .utils import highlight_hashtags, parse_datetime
+from .utils import parse_datetime
 from .widgets import Button, CheckBox, RadioButton
+from .richtext import ContentParser
 
 
 class Poll(urwid.ListBox):
@@ -85,8 +85,12 @@ def generate_poll_detail(self):
 
     def generate_contents(self, status):
         yield urwid.Divider()
-        for line in format_content(status.data["content"]):
-            yield urwid.Text(highlight_hashtags(line, set()))
+
+        parser = ContentParser()
+        widgetlist = parser.html_to_widgets(status.data["content"])
+
+        for line in widgetlist:
+            yield (line)
 
         yield urwid.Divider()
         yield self.build_linebox(self.generate_poll_detail())
diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py
index 3fb9b350..70192005 100644
--- a/toot/tui/timeline.py
+++ b/toot/tui/timeline.py
@@ -313,7 +313,6 @@ def remove_status(self, status):
 class StatusDetails(urwid.Pile):
     def __init__(self, timeline: Timeline, status: Optional[Status]):
         self.status = status
-        self.followed_tags = timeline.tui.followed_tags
         self.followed_accounts = timeline.tui.followed_accounts
 
         reblogged_by = status.author if status and status.reblog else None
diff --git a/toot/tui/utils.py b/toot/tui/utils.py
index 2f49362d..84cb7da6 100644
--- a/toot/tui/utils.py
+++ b/toot/tui/utils.py
@@ -81,21 +81,6 @@ def _gen():
     return list(_gen())
 
 
-def highlight_hashtags(line, followed_tags, attr="hashtag", followed_attr="followed_hashtag"):
-    hline = []
-
-    for p in re.split(HASHTAG_PATTERN, line):
-        if p.startswith("#"):
-            if p[1:].lower() in (t.lower() for t in followed_tags):
-                hline.append((followed_attr, p))
-            else:
-                hline.append((attr, p))
-        else:
-            hline.append(p)
-
-    return hline
-
-
 def show_media(paths):
     """
     Attempt to open an image viewer to show given media files.

From 3b92466acd208f4f08338a53368505ac5e1d4cf0 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Mon, 22 May 2023 17:55:01 -0400
Subject: [PATCH 47/55] HTML-to-markup conversion for status output to
 console/clipboard

This introduces a dependency on html2text. The library works
well for our limited use cases, but has not been updated since
2020.
---
 requirements.txt |  3 ++-
 setup.py         |  1 +
 toot/output.py   | 15 ++++++++++++++-
 toot/tui/app.py  | 25 ++++++++++++++++++++-----
 4 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 54b69dd3..8fec2ffe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,5 @@ requests>=2.13,<3.0
 beautifulsoup4>=4.5.0,<5.0
 wcwidth>=0.1.7
 urwid>=2.0.0,<3.0
-urwidgets=>=0.1,<0.2
+urwidgets>=0.1,<0.2
+html2text>=2020.1.16
diff --git a/setup.py b/setup.py
index 5abd1a44..94f6ac88 100644
--- a/setup.py
+++ b/setup.py
@@ -39,6 +39,7 @@
         "wcwidth>=0.1.7",
         "urwid>=2.0.0,<3.0",
         "urwidgets>=0.1,<0.2",
+        "html2text>=2020.1.16"
     ],
     entry_points={
         'console_scripts': [
diff --git a/toot/output.py b/toot/output.py
index c89aca99..4424cfb5 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -2,6 +2,7 @@
 import re
 import sys
 import textwrap
+import html2text
 
 from typing import List
 from wcwidth import wcswidth
@@ -267,6 +268,18 @@ def print_search_results(results):
 def print_status(status, width):
     reblog = status['reblog']
     content = reblog['content'] if reblog else status['content']
+
+    h2t = html2text.HTML2Text()
+
+    h2t.body_width = width
+    h2t.single_line_break = True
+    h2t.ignore_links = True
+    h2t.wrap_links = True
+    h2t.wrap_list_items = True
+    h2t.wrap_tables = True
+    
+    text_status = h2t.handle(content)
+
     media_attachments = reblog['media_attachments'] if reblog else status['media_attachments']
     in_reply_to = status['in_reply_to_id']
     poll = reblog.get('poll') if reblog else status.get('poll')
@@ -289,7 +302,7 @@ def print_status(status, width):
     )
 
     print_out("")
-    print_html(content, width)
+    print_out(highlight_hashtags(text_status))
 
     if media_attachments:
         print_out("\nMedia:")
diff --git a/toot/tui/app.py b/toot/tui/app.py
index 6dfeb306..6e9e6fee 100644
--- a/toot/tui/app.py
+++ b/toot/tui/app.py
@@ -1,5 +1,6 @@
 import logging
 import urwid
+import html2text
 
 from concurrent.futures import ThreadPoolExecutor
 
@@ -7,7 +8,6 @@
 from toot.console import get_default_visibility
 from toot.exceptions import ApiError
 from toot.commands import find_account
-
 from .compose import StatusComposer
 from .constants import PALETTE
 from .entities import Status
@@ -15,7 +15,7 @@
 from .overlays import StatusDeleteConfirmation, Account
 from .poll import Poll
 from .timeline import Timeline
-from .utils import parse_content_links, show_media, copy_to_clipboard
+from .utils import parse_content_links, show_media, copy_to_clipboard, parse_datetime
 
 logger = logging.getLogger(__name__)
 
@@ -620,9 +620,24 @@ def _done(loop):
         return self.run_in_thread(_delete, done_callback=_done)
 
     def copy_status(self, status):
-        # TODO: copy a better version of status content
-        # including URLs
-        copy_to_clipboard(self.screen, status.original.data["content"])
+        h2t = html2text.HTML2Text()
+        h2t.mark_code = True
+        h2t.single_line_break = True
+        h2t.body_width = 0  # nowrap
+
+        time = parse_datetime(status.original.data['created_at'])
+        time = time.strftime('%Y-%m-%d %H:%M %Z')
+
+        text_status = (f"[Status URL]({status.original.data['url']})\n\n"
+            + (status.original.author.display_name or "")
+            + "\n"
+            + (status.original.author.account or "")
+            + "\n\n"
+            + h2t.handle(status.original.data["content"])
+            + "\n\n"
+            + f"Created at: {time}")
+
+        copy_to_clipboard(self.screen, text_status)
         self.footer.set_message(f"Status {status.original.id} copied")
 
     # --- Overlay handling -----------------------------------------------------

From 2563f42efc781a9f39fef7f4c75f407ed08fe566 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 25 May 2023 20:03:36 -0400
Subject: [PATCH 48/55] leave URLs alone in copy to clipboard (don't use markup
 format)

---
 toot/output.py  | 1 +
 toot/tui/app.py | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/toot/output.py b/toot/output.py
index 4424cfb5..517b2cf0 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -277,6 +277,7 @@ def print_status(status, width):
     h2t.wrap_links = True
     h2t.wrap_list_items = True
     h2t.wrap_tables = True
+    h2t.ul_item_mark = "\N{bullet}"
     
     text_status = h2t.handle(content)
 
diff --git a/toot/tui/app.py b/toot/tui/app.py
index 6e9e6fee..cc925f18 100644
--- a/toot/tui/app.py
+++ b/toot/tui/app.py
@@ -621,14 +621,15 @@ def _done(loop):
 
     def copy_status(self, status):
         h2t = html2text.HTML2Text()
-        h2t.mark_code = True
-        h2t.single_line_break = True
         h2t.body_width = 0  # nowrap
+        h2t.single_line_break = True
+        h2t.ignore_links = True
+        h2t.ul_item_mark = "\N{bullet}"
 
         time = parse_datetime(status.original.data['created_at'])
         time = time.strftime('%Y-%m-%d %H:%M %Z')
 
-        text_status = (f"[Status URL]({status.original.data['url']})\n\n"
+        text_status = (f"{status.original.data['url']}\n\n"
             + (status.original.author.display_name or "")
             + "\n"
             + (status.original.author.account or "")

From 5c00edba3f687de115db7903f4d6b7b4ec6fabdd Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 25 May 2023 20:03:36 -0400
Subject: [PATCH 49/55] Strip extra newlines from status text output

This allows us to pass existing test_console.py tests
---
 toot/output.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/toot/output.py b/toot/output.py
index 517b2cf0..e9e8d97d 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -278,8 +278,8 @@ def print_status(status, width):
     h2t.wrap_list_items = True
     h2t.wrap_tables = True
     h2t.ul_item_mark = "\N{bullet}"
-    
-    text_status = h2t.handle(content)
+
+    text_status = h2t.handle(content).strip()
 
     media_attachments = reblog['media_attachments'] if reblog else status['media_attachments']
     in_reply_to = status['in_reply_to_id']

From bd375de357f902d0f051e39f4dbd4e78fa6abea9 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 25 May 2023 20:03:36 -0400
Subject: [PATCH 50/55] Strip excess newlines from status when copied to
 clipboard

---
 toot/tui/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/toot/tui/app.py b/toot/tui/app.py
index cc925f18..a747c7f2 100644
--- a/toot/tui/app.py
+++ b/toot/tui/app.py
@@ -634,7 +634,7 @@ def copy_status(self, status):
             + "\n"
             + (status.original.author.account or "")
             + "\n\n"
-            + h2t.handle(status.original.data["content"])
+            + h2t.handle(status.original.data["content"]).strip()
             + "\n\n"
             + f"Created at: {time}")
 

From 93a5306ff544b50679058aabbfe2db1f2f38b618 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 25 May 2023 20:03:36 -0400
Subject: [PATCH 51/55] HTML-to-markup conversion for status output to
 console/clipboard

This introduces a dependency on html2text. The library works
well for our limited use cases, but has not been updated since
2020.
---
 toot/output.py  | 1 +
 toot/tui/app.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/toot/output.py b/toot/output.py
index e9e8d97d..01c99d46 100644
--- a/toot/output.py
+++ b/toot/output.py
@@ -277,6 +277,7 @@ def print_status(status, width):
     h2t.wrap_links = True
     h2t.wrap_list_items = True
     h2t.wrap_tables = True
+    h2t.unicode_snob = True
     h2t.ul_item_mark = "\N{bullet}"
 
     text_status = h2t.handle(content).strip()
diff --git a/toot/tui/app.py b/toot/tui/app.py
index a747c7f2..c6e01cbb 100644
--- a/toot/tui/app.py
+++ b/toot/tui/app.py
@@ -624,6 +624,7 @@ def copy_status(self, status):
         h2t.body_width = 0  # nowrap
         h2t.single_line_break = True
         h2t.ignore_links = True
+        h2t.unicode_snob = True
         h2t.ul_item_mark = "\N{bullet}"
 
         time = parse_datetime(status.original.data['created_at'])

From 290bac2041d76bf4bb0eb76da353b3fa88116c16 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Thu, 25 May 2023 20:03:36 -0400
Subject: [PATCH 52/55] added test for timeline with (complex) HTML status
 message

---
 .vscode/launch.json   | 24 +++++++++++++++
 requirements-dev.txt  |  1 +
 tests/test_console.py | 70 +++++++++++++++++++++++++++++++++++++++++--
 toot/console.py       | 10 +++++++
 toot/debugger.py      | 10 +++++++
 5 files changed, 113 insertions(+), 2 deletions(-)
 create mode 100644 .vscode/launch.json
 create mode 100644 toot/debugger.py

diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 00000000..93f37d2e
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,24 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Remote Attach",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 9000
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "."
+                }
+            ],
+            "justMyCode": false
+        }
+    ]
+}
\ No newline at end of file
diff --git a/requirements-dev.txt b/requirements-dev.txt
index dfa5b15a..e3976a53 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -6,3 +6,4 @@ sphinx
 sphinx-autobuild
 twine
 wheel
+debugpy
diff --git a/tests/test_console.py b/tests/test_console.py
index ffe1d12c..2e458729 100644
--- a/tests/test_console.py
+++ b/tests/test_console.py
@@ -151,6 +151,74 @@ def test_timeline(mock_get, monkeypatch, capsys):
     assert err == ""
 
 
+@mock.patch('toot.http.get')
+def test_timeline_html_content(mock_get, monkeypatch, capsys):
+    mock_get.return_value = MockResponse([{
+        'id': '111111111111111111',
+        'account': {
+            'display_name': 'Frank Zappa 🎸',
+            'acct': 'fz'
+        },
+        'created_at': '2017-04-12T15:53:18.174Z',
+        'content': "<h2>HTML Render Test</h2><p><em>emphasized</em><br><u>underlined</u><br><strong>bold</strong><br><strong><em>bold and italic</em></strong><br><del>strikethrough</del><br>regular text</p><p>Code block:</p><pre><code>10 PRINT \"HELLO WORLD\"<br>20 GOTO 10<br></code></pre><blockquote><p>Something blockquoted here. The indentation is maintained as the text line wraps.</p></blockquote><ol><li>List item<ul><li>Nested item</li><li>Another nested </li></ul></li><li>Another list item. <ol><li>Something else nested</li><li>And a last nested</li></ol></li></ol><blockquote><p>Blockquote</p><ol><li>List in BQ</li><li>List item 2 in BQ</li></ol></blockquote><p><a href=\"https://babka.social/tags/hashtag\" class=\"mention hashtag\" rel=\"tag\">#<span>hashtag</span></a> <a href=\"https://babka.social/tags/test\" class=\"mention hashtag\" rel=\"tag\">#<span>test</span></a> <br><a href=\"https://a.com\" target=\"_blank\" rel=\"nofollow noopener noreferrer\"><span class=\"invisible\">https://</span><span class=\"\">a.com</span><span class=\"invisible\"></span></a> text after link</p>",
+        'reblog': None,
+        'in_reply_to_id': None,
+        'media_attachments': [],
+    }])
+
+    console.run_command(app, user, 'timeline', ['--once'])
+
+    mock_get.assert_called_once_with(app, user, '/api/v1/timelines/home', {'limit': 10})
+
+    out, err = capsys.readouterr()
+    lines = out.split("\n")  
+    reference = [
+        "────────────────────────────────────────────────────────────────────────────────────────────────────",
+        "Frank Zappa 🎸 @fz                                                              2017-04-12 15:53 UTC",
+        "",
+        "## HTML Render Test",
+        "",
+        " _emphasized_  ",
+        " _underlined_  ",
+        " **bold**  ",
+        " ** _bold and italic_**  ",
+        " ~~strikethrough~~  ",
+        "regular text",
+        "",
+        "Code block:",
+        "",
+        "    ",
+        "    10 PRINT \"HELLO WORLD\"  ",
+        "    20 GOTO 10  ",
+        "    ",
+        "> Something blockquoted here. The indentation is maintained as the text line wraps.",
+        "  1. List item",
+        "    • Nested item",
+        "    • Another nested ",
+        "  2. Another list item. ",
+        "    1. Something else nested",
+        "    2. And a last nested",
+        "",
+        "> Blockquote",
+        ">   1. List in BQ",
+        ">   2. List item 2 in BQ",
+        ">",
+        "",
+        "#hashtag #test  ",
+        "https://a.com text after link",
+        "",
+        "ID 111111111111111111   ",
+        "────────────────────────────────────────────────────────────────────────────────────────────────────",
+        "",
+    ]
+
+    assert len(lines) == len(reference)
+    for index, line in enumerate(lines):
+        assert line == reference[index], f"Line #{index}: Expected:\n{reference[index]}\nGot:\n{line}"
+
+    assert err == ""
+
+
 @mock.patch('toot.http.get')
 def test_timeline_with_re(mock_get, monkeypatch, capsys):
     mock_get.return_value = MockResponse([{
@@ -585,8 +653,6 @@ def test_notifications(mock_get, capsys):
         "────────────────────────────────────────────────────────────────────────────────────────────────────",
         "",
     ])
-
-
 @mock.patch('toot.http.get')
 def test_notifications_empty(mock_get, capsys):
     mock_get.return_value = MockResponse([])
diff --git a/toot/console.py b/toot/console.py
index 5c84d97c..6492473e 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -10,6 +10,7 @@
 from toot import config, commands, CLIENT_NAME, CLIENT_WEBSITE, __version__
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import print_out, print_err
+from .debugger import initialize_debugger
 
 VISIBILITY_CHOICES = ["public", "unlisted", "private", "direct"]
 VISIBILITY_CHOICES_STR = ", ".join(f"'{v}'" for v in VISIBILITY_CHOICES)
@@ -178,6 +179,12 @@ def editor(value):
         "action": 'store_true',
         "default": False,
     }),
+    (["--debugger"], {
+        "help": "launch with vscode debugpy",
+        "action": 'store_true',
+        "default": False,
+    }),
+
 ]
 
 # Arguments added to commands which require authentication
@@ -901,6 +908,9 @@ def run_command(app, user, name, args):
 
 
 def main():
+    if "--debugger" in sys.argv:
+        initialize_debugger()
+
     # Enable debug logging if --debug is in args
     if "--debug" in sys.argv:
         filename = os.getenv("TOOT_LOG_FILE")
diff --git a/toot/debugger.py b/toot/debugger.py
new file mode 100644
index 00000000..bc85d829
--- /dev/null
+++ b/toot/debugger.py
@@ -0,0 +1,10 @@
+def initialize_debugger():
+    import multiprocessing
+
+    if multiprocessing.current_process().pid > 1:
+        import debugpy
+
+        debugpy.listen(("0.0.0.0", 9000))
+        print("VSCode Debugger is ready to be attached on port 9000, press F5", flush=True)
+        debugpy.wait_for_client()
+        print("VSCode Debugger is now attached", flush=True)

From 474d1f4a46ce63014deff935720e6b2d5799528e Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Fri, 26 May 2023 17:53:15 -0400
Subject: [PATCH 53/55] Login to servers that don't honor the uri spec for
 V1::Instance

Pleroma, Akkoma, and other servers do not follow the Mastodon spec
for the 'uri' attribute which specifies that it contains the domain
name of the instance. Instead, they return a complete URI.

As a workaround, we now detect this situation and parse out the
domain from the URI when necessary. This fixes issue #347.

Thanks to @laleanor for their patch and @rjp for ideas on how to
make it work with GotoSocial and other servers
---
 toot/auth.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/toot/auth.py b/toot/auth.py
index 0ee2bace..db34d9fc 100644
--- a/toot/auth.py
+++ b/toot/auth.py
@@ -7,6 +7,7 @@
 from toot import api, config, DEFAULT_INSTANCE, User, App
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import print_out
+from urllib.parse import urlparse
 
 
 def register_app(domain, base_url):
@@ -46,8 +47,20 @@ def get_instance_domain(base_url):
         f"running Mastodon version <yellow>{instance['version']}</yellow>"
     )
 
+    # Pleroma and its forks return an actual URI here, rather than a
+    # domain name like Mastodon. This is contrary to the spec.¯
+    # in that case, parse out the domain and return it.
+
+    parsed_uri = urlparse(instance["uri"])
+
+    if parsed_uri.netloc:
+        # Pleroma, Akkoma, GotoSocial, etc.
+        return parsed_uri.netloc
+    else:
+        # Others including Mastodon servers
+        return parsed_uri.path
+
     # NB: when updating to v2 instance endpoint, this field has been renamed to `domain`
-    return instance["uri"]
 
 
 def create_user(app, access_token):

From 439aaa604e2478ed4ad117d2567ae27be6ee50cc Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Sat, 27 May 2023 21:50:17 -0400
Subject: [PATCH 54/55] Anchor tags with class = hashtag or mention_hashtag are
 highlighted

Previously this only worked for anchor tags with nested spans.
Now it works for anchor tags with or without nested spans.
---
 toot/tui/richtext.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index fc178be1..0cf326d9 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -238,6 +238,13 @@ def _a(self, tag) -> Tuple:
 
         attr = self.get_best_anchor_attr(attrib_list)
 
+        if attr == "a":
+            # didn't find an attribute to use
+            # in the child markup, so let's
+            # try the anchor tag's own attributes
+
+            attr = self.get_urwid_attr_name(tag)
+
         # hashtag anchors have a class of "mention hashtag"
         # or "hashtag"
         # we'll return style "class_mention_hashtag"

From 18ef70cd99e84a3e88c5c01e61a1fe7a12ec2ac4 Mon Sep 17 00:00:00 2001
From: Daniel Schwarz <schwarz.dan@gmail.com>
Date: Sat, 27 May 2023 21:51:41 -0400
Subject: [PATCH 55/55] URLEncode hrefs; fixes crash bug with href urls in
 foreign scripts

---
 .vscode/launch.json    | 24 ------------------------
 requirements-dev.txt   |  1 -
 toot/console.py        | 10 ----------
 toot/debugger.py       | 10 ----------
 toot/tui/richtext.py   |  3 +++
 toot/tui/timeline.py   |  3 ++-
 toot/utils/__init__.py | 12 ++++++++++++
 7 files changed, 17 insertions(+), 46 deletions(-)
 delete mode 100644 .vscode/launch.json
 delete mode 100644 toot/debugger.py

diff --git a/.vscode/launch.json b/.vscode/launch.json
deleted file mode 100644
index 93f37d2e..00000000
--- a/.vscode/launch.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Python: Remote Attach",
-            "type": "python",
-            "request": "attach",
-            "connect": {
-                "host": "localhost",
-                "port": 9000
-            },
-            "pathMappings": [
-                {
-                    "localRoot": "${workspaceFolder}",
-                    "remoteRoot": "."
-                }
-            ],
-            "justMyCode": false
-        }
-    ]
-}
\ No newline at end of file
diff --git a/requirements-dev.txt b/requirements-dev.txt
index e3976a53..dfa5b15a 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -6,4 +6,3 @@ sphinx
 sphinx-autobuild
 twine
 wheel
-debugpy
diff --git a/toot/console.py b/toot/console.py
index 6492473e..5c84d97c 100644
--- a/toot/console.py
+++ b/toot/console.py
@@ -10,7 +10,6 @@
 from toot import config, commands, CLIENT_NAME, CLIENT_WEBSITE, __version__
 from toot.exceptions import ApiError, ConsoleError
 from toot.output import print_out, print_err
-from .debugger import initialize_debugger
 
 VISIBILITY_CHOICES = ["public", "unlisted", "private", "direct"]
 VISIBILITY_CHOICES_STR = ", ".join(f"'{v}'" for v in VISIBILITY_CHOICES)
@@ -179,12 +178,6 @@ def editor(value):
         "action": 'store_true',
         "default": False,
     }),
-    (["--debugger"], {
-        "help": "launch with vscode debugpy",
-        "action": 'store_true',
-        "default": False,
-    }),
-
 ]
 
 # Arguments added to commands which require authentication
@@ -908,9 +901,6 @@ def run_command(app, user, name, args):
 
 
 def main():
-    if "--debugger" in sys.argv:
-        initialize_debugger()
-
     # Enable debug logging if --debug is in args
     if "--debug" in sys.argv:
         filename = os.getenv("TOOT_LOG_FILE")
diff --git a/toot/debugger.py b/toot/debugger.py
deleted file mode 100644
index bc85d829..00000000
--- a/toot/debugger.py
+++ /dev/null
@@ -1,10 +0,0 @@
-def initialize_debugger():
-    import multiprocessing
-
-    if multiprocessing.current_process().pid > 1:
-        import debugpy
-
-        debugpy.listen(("0.0.0.0", 9000))
-        print("VSCode Debugger is ready to be attached on port 9000, press F5", flush=True)
-        debugpy.wait_for_client()
-        print("VSCode Debugger is now attached", flush=True)
diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py
index 0cf326d9..175de59e 100644
--- a/toot/tui/richtext.py
+++ b/toot/tui/richtext.py
@@ -10,6 +10,7 @@
 from bs4.element import NavigableString, Tag
 from urwidgets import TextEmbed, Hyperlink, parse_text
 from urwid.util import decompose_tagmarkup
+from toot.utils import urlencode_url
 
 
 class ContentParser:
@@ -232,6 +233,8 @@ def _a(self, tag) -> Tuple:
         if not attrib_list:
             attrib_list = [tag]
         if href:
+            # urlencode the path and query portions of the URL
+            href = urlencode_url(href)
             # use ASCII ETX (end of record) as a
             # delimiter between the title and the HREF
             title += f"\x03{href}"
diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py
index 70192005..d8e20194 100644
--- a/toot/tui/timeline.py
+++ b/toot/tui/timeline.py
@@ -14,6 +14,7 @@
 from toot.tui import app
 from toot.tui.utils import time_ago
 from toot.utils.language import language_name
+from toot.utils import urlencode_url
 from urwidgets import Hyperlink, TextEmbed, parse_text
 
 logger = logging.getLogger("toot")
@@ -324,7 +325,7 @@ def linkify_content(self, text) -> urwid.Widget:
         TRANSFORM = {
             # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
             re.compile(r'(https?://[^\s]+)'):
-                lambda g: (len(g[1]), urwid.Filler(Hyperlink(g[1], "link"))),
+                lambda g: (len(g[1]), urwid.Filler(Hyperlink(urlencode_url(g[1]), "link", g[1]))),
         }
         markup_list = []
 
diff --git a/toot/utils/__init__.py b/toot/utils/__init__.py
index e8103acf..026c861f 100644
--- a/toot/utils/__init__.py
+++ b/toot/utils/__init__.py
@@ -5,6 +5,7 @@
 import tempfile
 import unicodedata
 import warnings
+from urllib.parse import urlparse, quote, unquote, urlencode
 
 from bs4 import BeautifulSoup
 from typing import Dict
@@ -81,6 +82,17 @@ def assert_domain_exists(domain):
         raise ConsoleError("Domain {} not found".format(domain))
 
 
+def urlencode_url(url):
+    parsed_url = urlparse(url)
+
+    # unencode before encoding, to prevent double-urlencoding
+    encoded_path = quote(unquote(parsed_url.path), safe="-._~()'!*:@,;+&=/")
+    encoded_query = urlencode({k: quote(unquote(v), safe="-._~()'!*:@,;?/") for k, v in parsed_url.params})
+    encoded_url = parsed_url._replace(path=encoded_path, params=encoded_query).geturl()
+
+    return encoded_url
+
+
 EOF_KEY = "Ctrl-Z" if os.name == 'nt' else "Ctrl-D"