src/node/serializing.rs: implement format_text

Author: niklak

CHANGELOG.md

@@ -7,7 +7,7 @@ All notable changes to the `dom_query` crate will be documented in this file.
 
 ### Added
 - Implemented `NodeRef::normalized_char_count` which estimates the number of characters in the text of descendant nodes as if the total string were normalized.
-
+- Implemented `Document::formatted_text`, `Selection::formatted_text`, and `NodeRef::formatted_text`, which return formatted text of the document, selection, or node respectively.
 
 ## [0.12.0] - 2025-01-16
 

src/document.rs

@@ -117,6 +117,17 @@ impl Document {
         self.root().text()
     }
 
+    /// Returns the formatted text of the document and its descendants. This is the same as
+    /// the `text()` method, but with a few differences:
+    ///
+    /// - Whitespace is normalized so that there is only one space between words.
+    /// - All whitespace is removed from the beginning and end of the text.
+    /// - After block elements, a double newline is added.
+    /// - For elements like `br`, 'hr', 'li', 'tr' a single newline is added.
+    pub fn formatted_text(&self) -> StrTendril {
+        self.root().formatted_text()
+    }
+
     /// Finds the base URI of the tree by looking for `<base>` tags in document's head.
     ///
     /// The base URI is the value of the `href` attribute of the first

src/node.rs

@@ -15,6 +15,7 @@ pub use iters::{
 };
 pub use node_data::{Element, NodeData};
 pub use node_ref::{Node, NodeRef};
+pub(crate) use serializing::format_text;
 pub use serializing::SerializableNodeRef;
 
 /// Represents a Node ID.

src/node/node_ref.rs

@@ -21,6 +21,7 @@ use super::child_nodes;
 use super::id_provider::NodeIdProver;
 use super::inner::TreeNode;
 use super::node_data::NodeData;
+use super::serializing::format_text;
 use super::serializing::SerializableNodeRef;
 use super::NodeId;
 
@@ -552,6 +553,18 @@ impl NodeRef<'_> {
         TreeNodeOps::immediate_text_of(nodes, self.id)
     }
 
+    /// Returns the formatted text of the node and its descendants. This is the same as
+    /// the `text()` method, but with a few differences:
+    ///
+    /// - Whitespace is normalized so that there is only one space between words.
+    /// - All whitespace is removed from the beginning and end of the text.
+    /// - After block elements, a double newline is added.
+    /// - For elements like `br`, 'hr', 'li', 'tr' a single newline is added.
+
+    pub fn formatted_text(&self) -> StrTendril {
+        format_text(self, false)
+    }
+
     /// Checks if the node contains the specified text
     pub fn has_text(&self, needle: &str) -> bool {
         let mut ops = vec![self.id];

src/node/serializing.rs

@@ -3,7 +3,11 @@ use std::io;
 
 use html5ever::serialize::TraversalScope;
 use html5ever::serialize::{Serialize, Serializer};
-use html5ever::QualName;
+
+use html5ever::{local_name, QualName};
+use tendril::StrTendril;
+
+use crate::TreeNodeOps;
 
 use super::node_data::NodeData;
 use super::node_ref::NodeRef;
@@ -82,3 +86,122 @@ impl Serialize for SerializableNodeRef<'_> {
         Ok(())
     }
 }
+
+pub(crate) fn format_text(root_node: &NodeRef, include_node: bool) -> StrTendril {
+    let id = root_node.id;
+    let nodes = root_node.tree.nodes.borrow();
+    let mut ops = if include_node {
+        vec![SerializeOp::Open(id)]
+    } else {
+        child_nodes(Ref::clone(&nodes), &id, true)
+            .map(SerializeOp::Open)
+            .collect()
+    };
+
+    let mut text = StrTendril::new();
+
+    while let Some(op) = ops.pop() {
+        match op {
+            SerializeOp::Open(id) => {
+                let node = match nodes.get(id.value) {
+                    Some(node) => node,
+                    None => continue,
+                };
+
+                match node.data {
+                    NodeData::Text { ref contents } => {
+                        if contents.is_empty() {
+                            continue;
+                        }
+                        let follows_newline = text.ends_with('\n') || text.is_empty();
+                        let normalized = normalize_text(contents.as_ref(), follows_newline);
+                        text.push_tendril(&normalized);
+                    }
+                    NodeData::Element(ref e) => {
+                        ops.push(SerializeOp::Close(&e.name));
+
+                        if matches!(e.name.local, local_name!("pre")) {
+                            text.push_tendril(&TreeNodeOps::text_of(Ref::clone(&nodes), id));
+                            continue;
+                        }
+
+                        ops.extend(
+                            child_nodes(Ref::clone(&nodes), &id, true).map(SerializeOp::Open),
+                        );
+                    }
+                    NodeData::Document | NodeData::Fragment => {
+                        // Push children in reverse order
+                        ops.extend(
+                            child_nodes(Ref::clone(&nodes), &id, true).map(SerializeOp::Open),
+                        );
+                        continue;
+                    }
+                    _ => {}
+                }
+            }
+            SerializeOp::Close(name) => {
+                if text.ends_with("\n\n") {
+                    continue;
+                }
+                if matches!(
+                    name.local,
+                    local_name!("article")
+                        | local_name!("blockquote")
+                        | local_name!("section")
+                        | local_name!("div")
+                        | local_name!("p")
+                        | local_name!("pre")
+                        | local_name!("h1")
+                        | local_name!("h2")
+                        | local_name!("h3")
+                        | local_name!("h4")
+                        | local_name!("h5")
+                        | local_name!("h6")
+                        | local_name!("ul")
+                        | local_name!("ol")
+                        | local_name!("dl")
+                        | local_name!("table")
+                ) {
+                    text.push_slice("\n\n");
+                } else if matches!(
+                    name.local,
+                    local_name!("br") | local_name!("hr") | local_name!("li") | local_name!("tr")
+                ) {
+                    text.push_char('\n');
+                }
+            }
+        }
+    }
+    if !include_node {
+        while !text.is_empty() && text.ends_with(char::is_whitespace) {
+            text.pop_back(1);
+        }
+    }
+    text
+}
+
+fn normalize_text(text: &str, follows_newline: bool) -> StrTendril {
+    let push_start_whitespace = !follows_newline && text.starts_with(char::is_whitespace);
+    let push_end_whitespace = text.ends_with(char::is_whitespace);
+
+    let mut result = StrTendril::with_capacity(text.len() as u32);
+    let mut iter = text.split_whitespace();
+
+    if let Some(first) = iter.next() {
+        if push_start_whitespace {
+            result.push_char(' ');
+        }
+        result.push_slice(first);
+        for word in iter {
+            result.push_char(' ');
+            result.push_slice(word);
+        }
+    }
+    if result.is_empty() {
+        return result;
+    }
+    if push_end_whitespace {
+        result.push_char(' ');
+    }
+    result
+}

src/selection.rs

@@ -7,7 +7,7 @@ use tendril::StrTendril;
 
 use crate::document::Document;
 use crate::matcher::{MatchScope, Matcher, Matches};
-use crate::node::{ancestor_nodes, child_nodes, NodeId, NodeRef, TreeNode};
+use crate::node::{ancestor_nodes, child_nodes, format_text, NodeId, NodeRef, TreeNode};
 use crate::{Tree, TreeNodeOps};
 
 /// Selection represents a collection of nodes matching some criteria. The
@@ -176,6 +176,21 @@ impl Selection<'_> {
     pub fn immediate_text(&self) -> StrTendril {
         self.text_fn(TreeNodeOps::immediate_text_of)
     }
+
+    /// Returns the formatted text of the selected nodes and their descendants.
+    /// This is the same as the `text()` method, but with a few differences:
+    ///
+    /// - Whitespace is normalized so that there is only one space between words.
+    /// - All whitespace is removed from the beginning and end of the text.
+    /// - After block elements, a double newline is added.
+    /// - For elements like `br`, 'hr', 'li', 'tr' a single newline is added.
+    pub fn formatted_text(&self) -> StrTendril {
+        let mut s = StrTendril::new();
+        for node in self.nodes() {
+            s.push_tendril(&format_text(node, true));
+        }
+        s
+    }
 }
 
 //matching methods

tests/data.rs

@@ -104,3 +104,25 @@ pub static ATTRS_CONTENTS: &str = r#"<!DOCTYPE html>
             </div>
         </body>
     </html>"#;
+
+pub static DMC_CONTENTS: &str = r#"<!DOCTYPE html>
+<html>
+    <head></head>
+    <body>
+        <div id="main">
+            <div>
+                <p>Listen up y'all, it's time to get down</p>
+                <p>'Bout that <b>normalized_char_count</b> in this town</p>
+                <p>Traversing nodes with style and grace</p>
+                <p>Counting chars at a steady pace</p>
+            </div>
+
+            <div>
+                <p>No split whitespace, that's old school</p>
+                <p>Direct counting's our golden rule</p>
+                <p>Skip them nodes that ain't text or element</p>
+                <p>That's how we keep our code development!</p>
+            </div>
+        </div>
+    </body>
+</html>"#;

tests/node-traversal.rs

@@ -1,6 +1,6 @@
 mod data;
 
-use data::ANCESTORS_CONTENTS;
+use data::{ANCESTORS_CONTENTS, DMC_CONTENTS};
 use dom_query::{Document, NodeData, Selection};
 
 #[cfg(target_arch = "wasm32")]
@@ -328,3 +328,26 @@ fn test_node_normalized_char_count() {
     let got = main_node.normalized_char_count();
     assert_eq!(got, expected);
 }
+
+#[cfg_attr(not(target_arch = "wasm32"), test)]
+#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+fn test_doc_formatted_text() {
+    let doc = Document::from(DMC_CONTENTS);
+    let text = doc.formatted_text();
+    let expected = r#"Listen up y'all, it's time to get down
+
+'Bout that normalized_char_count in this town
+
+Traversing nodes with style and grace
+
+Counting chars at a steady pace
+
+No split whitespace, that's old school
+
+Direct counting's our golden rule
+
+Skip them nodes that ain't text or element
+
+That's how we keep our code development!"#;
+    assert_eq!(text.as_ref(), expected);
+}

tests/selection-traversal.rs

@@ -2,6 +2,7 @@ mod data;
 
 use data::doc;
 use data::doc_wiki;
+use data::DMC_CONTENTS;
 use data::{ANCESTORS_CONTENTS, LIST_CONTENTS};
 use dom_query::Document;
 
@@ -522,3 +523,29 @@ fn test_selection_get_node() {
     let third = sel.get(2);
     assert!(third.is_none());
 }
+
+#[cfg_attr(not(target_arch = "wasm32"), test)]
+#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+fn test_selection_formatted_text() {
+    let doc = Document::from(DMC_CONTENTS);
+    let sel = doc.select("p");
+    let text = sel.formatted_text();
+    let expected = r#"Listen up y'all, it's time to get down
+
+'Bout that normalized_char_count in this town
+
+Traversing nodes with style and grace
+
+Counting chars at a steady pace
+
+No split whitespace, that's old school
+
+Direct counting's our golden rule
+
+Skip them nodes that ain't text or element
+
+That's how we keep our code development!
+
+"#;
+    assert_eq!(text.as_ref(), expected);
+}