feat: create SymbolIterator for block parsing

commons/src/scanner/mod.rs

@@ -1,4 +1,5 @@
-//! Scanner and helper types and traits for structurization of Unimarkup input.
+//! Functionality, iterators, helper types and traits to get [`Symbol`]s from `&str`.
+//! These [`Symbol`]s and iterators are used to convert the input into a Unimarkup document.
 
 use icu_segmenter::GraphemeClusterSegmenter;
 
@@ -9,87 +10,65 @@ mod symbol;
 use position::{Offset, Position as SymPos};
 pub use symbol::{iterator::*, Symbol, SymbolKind};
 
-#[derive(Debug)]
-pub struct Scanner {
-    segmenter: GraphemeClusterSegmenter,
-}
+/// Scans given input and returns vector of [`Symbol`]s needed to convert the input to Unimarkup content.
+pub fn scan_str(input: &str) -> Vec<Symbol<'_>> {
+    let segmenter = GraphemeClusterSegmenter::new();
 
-impl Clone for Scanner {
-    fn clone(&self) -> Self {
-        Scanner::new()
-    }
-}
-
-impl Default for Scanner {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl Scanner {
-    pub fn new() -> Self {
-        let segmenter = GraphemeClusterSegmenter::new();
-
-        Self { segmenter }
-    }
+    let mut symbols: Vec<Symbol> = Vec::new();
+    let mut curr_pos: SymPos = SymPos::default();
+    let mut prev_offset = 0;
 
-    pub fn scan_str<'s>(&self, input: &'s str) -> Vec<Symbol<'s>> {
-        let mut symbols: Vec<Symbol> = Vec::new();
-        let mut curr_pos: SymPos = SymPos::default();
-        let mut prev_offset = 0;
+    // skip(1) to ignore break at start of input
+    for offset in segmenter.segment_str(input).skip(1) {
+        if let Some(grapheme) = input.get(prev_offset..offset) {
+            let mut kind = SymbolKind::from(grapheme);
 
-        // skip(1) to ignore break at start of input
-        for offset in self.segmenter.segment_str(input).skip(1) {
-            if let Some(grapheme) = input.get(prev_offset..offset) {
-                let mut kind = SymbolKind::from(grapheme);
-
-                let end_pos = if kind == SymbolKind::Newline {
-                    SymPos {
-                        line: (curr_pos.line + 1),
-                        ..Default::default()
-                    }
-                } else {
-                    SymPos {
-                        line: curr_pos.line,
-                        col_utf8: (curr_pos.col_utf8 + grapheme.len()),
-                        col_utf16: (curr_pos.col_utf16 + grapheme.encode_utf16().count()),
-                        col_grapheme: (curr_pos.col_grapheme + 1),
-                    }
-                };
-
-                if curr_pos.col_utf8 == 1 && kind == SymbolKind::Newline {
-                    // newline at the start of line -> Blankline
-                    kind = SymbolKind::Blankline;
+            let end_pos = if kind == SymbolKind::Newline {
+                SymPos {
+                    line: (curr_pos.line + 1),
+                    ..Default::default()
                 }
+            } else {
+                SymPos {
+                    line: curr_pos.line,
+                    col_utf8: (curr_pos.col_utf8 + grapheme.len()),
+                    col_utf16: (curr_pos.col_utf16 + grapheme.encode_utf16().count()),
+                    col_grapheme: (curr_pos.col_grapheme + 1),
+                }
+            };
 
-                symbols.push(Symbol {
-                    input,
-                    kind,
-                    offset: Offset {
-                        start: prev_offset,
-                        end: offset,
-                    },
-                    start: curr_pos,
-                    end: end_pos,
-                });
-
-                curr_pos = end_pos;
+            if curr_pos.col_utf8 == 1 && kind == SymbolKind::Newline {
+                // newline at the start of line -> Blankline
+                kind = SymbolKind::Blankline;
             }
-            prev_offset = offset;
-        }
-
-        symbols.push(Symbol {
-            input,
-            kind: SymbolKind::EOI,
-            offset: Offset {
-                start: prev_offset,
-                end: prev_offset,
-            },
-            start: curr_pos,
-            end: curr_pos,
-        });
 
-        // last offset not needed, because break at EOI is always available
-        symbols
+            symbols.push(Symbol {
+                input,
+                kind,
+                offset: Offset {
+                    start: prev_offset,
+                    end: offset,
+                },
+                start: curr_pos,
+                end: end_pos,
+            });
+
+            curr_pos = end_pos;
+        }
+        prev_offset = offset;
     }
+
+    symbols.push(Symbol {
+        input,
+        kind: SymbolKind::EOI,
+        offset: Offset {
+            start: prev_offset,
+            end: prev_offset,
+        },
+        start: curr_pos,
+        end: curr_pos,
+    });
+
+    // last offset not needed, because break at EOI is always available
+    symbols
 }

commons/src/scanner/symbol/iterator/matcher.rs

@@ -11,6 +11,7 @@ use super::SymbolIterator;
 
 /// Function type to notify an iterator if an end was reached.
 pub type IteratorEndFn = Rc<dyn (Fn(&mut dyn EndMatcher) -> bool)>;
+
 /// Function type to consume prefix sequences of a new line.
 pub type IteratorPrefixFn = Rc<dyn (Fn(&mut dyn PrefixMatcher) -> bool)>;
 
@@ -103,7 +104,7 @@ impl<'input> EndMatcher for SymbolIterator<'input> {
         let is_empty_line = self.is_empty_line();
 
         if is_empty_line {
-            self.set_curr_index(self.peek_index()); // To consume peeked symbols
+            self.set_index(self.peek_index()); // To consume peeked symbols
         }
 
         is_empty_line
@@ -126,21 +127,20 @@ impl<'input> EndMatcher for SymbolIterator<'input> {
         let matched = self.matches(sequence);
 
         if matched {
-            self.set_curr_index(self.peek_index()); // To consume peeked symbols
+            self.set_index(self.peek_index()); // To consume peeked symbols
         }
 
         matched
     }
 
     fn at_depth(&self, depth: usize) -> bool {
-        self.curr_depth() == depth
+        self.depth() == depth
     }
 }
 
 impl<'input> PrefixMatcher for SymbolIterator<'input> {
     fn consumed_prefix(&mut self, sequence: &[SymbolKind]) -> bool {
-        #[cfg(debug_assertions)]
-        assert!(
+        debug_assert!(
             !sequence.contains(&SymbolKind::Newline),
             "Newline symbol in prefix match is not allowed."
         );

commons/src/scanner/symbol/iterator/mod.rs

@@ -16,6 +16,10 @@ pub use root::*;
 /// It allows to add matcher functions to notify the iterator,
 /// when an end of an element is reached, or what prefixes to strip on a new line.
 /// Additionaly, the iterator may be nested to enable transparent iterating for nested elements.
+///
+/// *Transparent* meaning that the nested iterator does not see [`Symbol`]s consumed by the wrapped (parent) iterator.
+/// In other words, wrapped iterators control which [`Symbol`]s will be passed to their nested iterator.
+/// Therefore, each nested iterator only sees those [`Symbol`]s that are relevant to its scope.
 #[derive(Clone)]
 pub struct SymbolIterator<'input> {
     /// The [`SymbolIteratorKind`] of this iterator.
@@ -33,8 +37,6 @@ pub struct SymbolIterator<'input> {
 }
 
 /// The [`SymbolIteratorKind`] defines the kind of a [`SymbolIterator`].
-///
-/// **Note:** This enables iterator nesting.
 #[derive(Clone)]
 pub enum SymbolIteratorKind<'input> {
     /// Defines an iterator as being nested.
@@ -94,25 +96,25 @@ impl<'input> SymbolIterator<'input> {
 
     /// The current nested depth this iterator is at.
     /// The root iterator starts at 0, and every iterator created using [`Self::nest()`] is one depth higher than its parent.
-    pub fn curr_depth(&self) -> usize {
+    pub fn depth(&self) -> usize {
         self.depth
     }
 
     /// Returns the current index this iterator is in the [`Symbol`] slice of the root iterator.
-    pub fn curr_index(&self) -> usize {
+    pub fn index(&self) -> usize {
         match &self.kind {
-            SymbolIteratorKind::Nested(parent) => parent.curr_index(),
-            SymbolIteratorKind::Root(root) => root.curr_index,
+            SymbolIteratorKind::Nested(parent) => parent.index(),
+            SymbolIteratorKind::Root(root) => root.index,
         }
     }
 
     /// Sets the current index of this iterator to the given index.
-    pub fn set_curr_index(&mut self, index: usize) {
+    pub(super) fn set_index(&mut self, index: usize) {
         if index >= self.start_index {
             match self.kind.borrow_mut() {
-                SymbolIteratorKind::Nested(parent) => parent.set_curr_index(index),
+                SymbolIteratorKind::Nested(parent) => parent.set_index(index),
                 SymbolIteratorKind::Root(root) => {
-                    root.curr_index = index;
+                    root.index = index;
                     root.peek_index = index;
                 }
             }
@@ -128,8 +130,8 @@ impl<'input> SymbolIterator<'input> {
     }
 
     /// Sets the peek index of this iterator to the given index.
-    pub fn set_peek_index(&mut self, index: usize) {
-        if index >= self.curr_index() {
+    fn set_peek_index(&mut self, index: usize) {
+        if index >= self.index() {
             match self.kind.borrow_mut() {
                 SymbolIteratorKind::Nested(parent) => parent.set_peek_index(index),
                 SymbolIteratorKind::Root(root) => {
@@ -143,7 +145,7 @@ impl<'input> SymbolIterator<'input> {
     ///
     /// **Note:** Needed to reset peek index after using `peeking_next()`.
     pub fn reset_peek(&mut self) {
-        self.set_peek_index(self.curr_index());
+        self.set_peek_index(self.index());
     }
 
     /// Returns the maximal remaining symbols in this iterator.
@@ -186,7 +188,7 @@ impl<'input> SymbolIterator<'input> {
     ) -> SymbolIterator<'input> {
         SymbolIterator {
             kind: SymbolIteratorKind::Nested(Box::new(self.clone())),
-            start_index: self.curr_index(),
+            start_index: self.index(),
             depth: self.depth + 1,
             prefix_match,
             end_match,
@@ -199,6 +201,13 @@ impl<'input> SymbolIterator<'input> {
     /// **Note:** Only updates the parent if `self` is nested.
     pub fn update(self, parent: &mut Self) {
         if let SymbolIteratorKind::Nested(self_parent) = self.kind {
+            // Make sure it actually is the parent.
+            // It is not possible to check more precisely, because other indices are expected to be different due to `clone()`.
+            debug_assert_eq!(
+                self_parent.start_index, parent.start_index,
+                "Updated iterator is not the actual parent of this iterator."
+            );
+
             *parent = *self_parent;
         }
     }
@@ -233,21 +242,11 @@ impl<'input> SymbolIterator<'input> {
     }
 }
 
-impl<'input> From<&'input [Symbol<'input>]> for SymbolIterator<'input> {
-    fn from(value: &'input [Symbol<'input>]) -> Self {
-        SymbolIterator {
-            kind: SymbolIteratorKind::Root(SymbolIteratorRoot::from(value)),
-            start_index: 0,
-            depth: 0,
-            prefix_match: None,
-            end_match: None,
-            iter_end: false,
-        }
-    }
-}
-
-impl<'input> From<&'input Vec<Symbol<'input>>> for SymbolIterator<'input> {
-    fn from(value: &'input Vec<Symbol<'input>>) -> Self {
+impl<'input, T> From<T> for SymbolIterator<'input>
+where
+    T: Into<&'input [Symbol<'input>]>,
+{
+    fn from(value: T) -> Self {
         SymbolIterator {
             kind: SymbolIteratorKind::Root(SymbolIteratorRoot::from(value)),
             start_index: 0,
@@ -322,21 +321,21 @@ mod test {
 
     use itertools::{Itertools, PeekingNext};
 
-    use crate::scanner::{PrefixMatcher, Scanner, SymbolKind};
+    use crate::scanner::{PrefixMatcher, SymbolKind};
 
     use super::SymbolIterator;
 
     #[test]
     fn peek_while_index() {
-        let symbols = Scanner::new().scan_str("## ");
+        let symbols = crate::scanner::scan_str("## ");
 
-        let mut iterator = SymbolIterator::from(&symbols);
+        let mut iterator = SymbolIterator::from(&*symbols);
         let hash_cnt = iterator
             .peeking_take_while(|symbol| symbol.kind == SymbolKind::Hash)
             .count();
 
         let next_symbol = iterator.nth(hash_cnt);
-        let curr_index = iterator.curr_index();
+        let curr_index = iterator.index();
 
         assert_eq!(hash_cnt, 2, "Hash symbols in input not correctly detected.");
         assert_eq!(curr_index, 3, "Current index was not updated correctly.");
@@ -353,14 +352,14 @@ mod test {
 
     #[test]
     fn peek_next() {
-        let symbols = Scanner::new().scan_str("#*");
+        let symbols = crate::scanner::scan_str("#*");
 
-        let mut iterator = SymbolIterator::from(&symbols);
+        let mut iterator = SymbolIterator::from(&*symbols);
 
         let peeked_symbol = iterator.peeking_next(|_| true);
         let next_symbol = iterator.next();
         let next_peeked_symbol = iterator.peeking_next(|_| true);
-        let curr_index = iterator.curr_index();
+        let curr_index = iterator.index();
 
         assert_eq!(curr_index, 1, "Current index was not updated correctly.");
         assert_eq!(
@@ -387,9 +386,9 @@ mod test {
 
     #[test]
     fn reach_end() {
-        let symbols = Scanner::new().scan_str("text*");
+        let symbols = crate::scanner::scan_str("text*");
 
-        let mut iterator = SymbolIterator::from(&symbols).nest(
+        let mut iterator = SymbolIterator::from(&*symbols).nest(
             None,
             Some(Rc::new(|matcher| matcher.matches(&[SymbolKind::Star]))),
         );
@@ -415,7 +414,7 @@ mod test {
 
     #[test]
     fn with_nested_and_parent_prefix() {
-        let symbols = Scanner::new().scan_str("a\n* *b");
+        let symbols = crate::scanner::scan_str("a\n* *b");
 
         let iterator = SymbolIterator::with(
             &symbols,
@@ -452,7 +451,7 @@ mod test {
 
     #[test]
     fn depth_matcher() {
-        let symbols = Scanner::new().scan_str("[o [i]]");
+        let symbols = crate::scanner::scan_str("[o [i]]");
 
         let mut iterator = SymbolIterator::with(
             &symbols,