use inlinelist for bufio

Author: thatstoasty

gojo/bufio/bufio.mojo

@@ -30,7 +30,9 @@ struct Scanner[R: io.Reader]():
     var split: SplitFunction  # The function to split the tokens.
     var max_token_size: Int  # Maximum size of a token; modified by tests.
     var token: List[Byte]  # Last token returned by split.
-    var buf: List[Byte]  # buffer used as argument to split.
+    var data: UnsafePointer[UInt8]  # contents are the bytes buf[off : len(buf)]
+    var size: Int
+    var capacity: Int
     var start: Int  # First non-processed byte in buf.
     var end: Int  # End of data in buf.
     var empties: Int  # Count of successive empty tokens.
@@ -44,7 +46,9 @@ struct Scanner[R: io.Reader]():
         split: SplitFunction = scan_lines,
         max_token_size: Int = MAX_SCAN_TOKEN_SIZE,
         token: List[Byte] = List[Byte](capacity=io.BUFFER_SIZE),
-        buf: List[Byte] = List[Byte](capacity=io.BUFFER_SIZE),
+        data: UnsafePointer[UInt8] = UnsafePointer[UInt8](),
+        size: Int = 0,
+        capacity: Int = io.BUFFER_SIZE,
         start: Int = 0,
         end: Int = 0,
         empties: Int = 0,
@@ -55,14 +59,42 @@ struct Scanner[R: io.Reader]():
         self.split = split
         self.max_token_size = max_token_size
         self.token = token
-        self.buf = buf
+        self.data = data
+        self.size = size
+        self.capacity = capacity
         self.start = start
         self.end = end
         self.empties = empties
         self.scan_called = scan_called
         self.done = done
         self.err = Error()
 
+    @always_inline
+    fn _resize(inout self, capacity: Int) -> None:
+        """
+        Resizes the string builder buffer.
+
+        Args:
+          capacity: The new capacity of the string builder buffer.
+        """
+        var new_data = UnsafePointer[UInt8]().alloc(capacity)
+        memcpy(new_data, self.data, self.size)
+        self.data.free()
+        self.data = new_data
+        self.capacity = capacity
+
+        return None
+
+    @always_inline
+    fn __del__(owned self):
+        if self.data:
+            self.data.free()
+
+    @always_inline
+    fn as_bytes_slice(self: Reference[Self]) -> Span[UInt8, self.is_mutable, self.lifetime]:
+        """Returns the internal buffer data as a Span[UInt8]."""
+        return Span[UInt8, self.is_mutable, self.lifetime](unsafe_ptr=self[].data, len=self[].size)
+
     fn current_token_as_bytes(self) -> List[Byte]:
         """Returns the most recent token generated by a call to [Scanner.Scan].
         The underlying array may point to data that will be overwritten
@@ -73,7 +105,9 @@ struct Scanner[R: io.Reader]():
     fn current_token(self) -> String:
         """Returns the most recent token generated by a call to [Scanner.Scan]
         as a newly allocated string holding its bytes."""
-        return String(self.token)
+        var copy = self.token
+        copy.append(0)
+        return String(copy)
 
     fn scan(inout self) raises -> Bool:
         """Advances the [Scanner] to the next token, which will then be
@@ -101,7 +135,7 @@ struct Scanner[R: io.Reader]():
                 var at_eof = False
                 if self.err:
                     at_eof = True
-                advance, token, err = self.split(self.buf[self.start : self.end], at_eof)
+                advance, token, err = self.split(self.as_bytes_slice()[self.start : self.end], at_eof)
                 if err:
                     if str(err) == str(ERR_FINAL_TOKEN):
                         self.token = token
@@ -140,27 +174,24 @@ struct Scanner[R: io.Reader]():
             # Must read more data.
             # First, shift data to beginning of buffer if there's lots of empty space
             # or space is needed.
-            if self.start > 0 and (self.end == len(self.buf) or self.start > int(len(self.buf) / 2)):
-                _ = copy(self.buf, self.buf[self.start : self.end])
+            if self.start > 0 and (self.end == self.size or self.start > int(self.size / 2)):
+                self.data = self.as_bytes_slice()[self.start : self.end].unsafe_ptr()
                 self.end -= self.start
                 self.start = 0
 
             # Is the buffer full? If so, resize.
-            if self.end == len(self.buf):
+            if self.end == self.size:
                 # Guarantee no overflow in the multiplication below.
-                if len(self.buf) >= self.max_token_size or len(self.buf) > int(MAX_INT / 2):
-                    self.set_err(Error(str(ERR_TOO_LONG)))
+                if self.size >= self.max_token_size or self.size > int(MAX_INT / 2):
+                    self.set_err(ERR_TOO_LONG)
                     return False
 
-                var new_size = len(self.buf) * 2
+                var new_size = self.size * 2
                 if new_size == 0:
                     new_size = START_BUF_SIZE
 
-                # Make a new List[Byte] buffer and copy the elements in
                 new_size = min(new_size, self.max_token_size)
-                var new_buf = List[Byte](capacity=new_size)
-                _ = copy(new_buf, self.buf[self.start : self.end])
-                self.buf = new_buf
+                self._resize(new_size)
                 self.end -= self.start
                 self.start = 0
 
@@ -170,13 +201,13 @@ struct Scanner[R: io.Reader]():
             var loop = 0
             while True:
                 var bytes_read: Int
-                var sl = self.buf[self.end : len(self.buf)]
+                var sl = List[UInt8](self.as_bytes_slice()[self.end : self.size])
                 var err: Error
 
                 # Catch any reader errors and set the internal error field to that err instead of bubbling it up.
                 bytes_read, err = self.reader.read(sl)
-                _ = copy(self.buf, sl, self.end)
-                if bytes_read < 0 or len(self.buf) - self.end < bytes_read:
+                self.data = sl.steal_data()
+                if bytes_read < 0 or self.size - self.end < bytes_read:
                     self.set_err(Error(str(ERR_BAD_READ_COUNT)))
                     break
 

gojo/bufio/scan.mojo

@@ -1,3 +1,6 @@
+from collections import InlineList
+
+
 fn copy[T: CollectionElement](inout target: List[T], source: List[T], start: Int = 0) -> Int:
     """Copies the contents of source into target at the same index. Returns the number of bytes copied.
     Added a start parameter to specify the index to start copying into.
@@ -46,6 +49,30 @@ fn copy[T: CollectionElement](inout target: List[T], source: Span[T], start: Int
     return count
 
 
+fn copy[T: CollectionElement](inout list: InlineList[T], source: Span[T], start: Int = 0) -> Int:
+    """Copies the contents of source into target at the same index. Returns the number of bytes copied.
+    Added a start parameter to specify the index to start copying into.
+
+    Args:
+        list: The buffer to copy into.
+        source: The buffer to copy from.
+        start: The index to start copying into.
+
+    Returns:
+        The number of bytes copied.
+    """
+    var count = 0
+
+    for i in range(len(source)):
+        if i + start > len(list):
+            list[i + start] = source[i]
+        else:
+            list.append(source[i])
+        count += 1
+
+    return count
+
+
 fn copy(
     inout target: List[UInt8],
     source: DTypePointer[DType.uint8],
@@ -78,27 +105,6 @@ fn copy(
     return count
 
 
-# fn copy[T: CollectionElement](inout target: Span[T], source: Span[T], start: Int = 0) -> Int:
-#     """Copies the contents of source into target at the same index. Returns the number of bytes copied.
-#     Added a start parameter to specify the index to start copying into.
-
-#     Args:
-#         target: The buffer to copy into.
-#         source: The buffer to copy from.
-#         start: The index to start copying into.
-
-#     Returns:
-#         The number of bytes copied.
-#     """
-#     var count = 0
-
-#     for i in range(len(source)):
-#         target[i + start] = source[i]
-#         count += 1
-
-#     return count
-
-
 fn cap[T: CollectionElement](iterable: List[T]) -> Int:
     """Returns the capacity of the List.
 

gojo/builtins/attributes.mojo

@@ -485,5 +485,4 @@ fn new_buffer(owned s: String) -> Buffer:
     Returns:
         A new [Buffer] initialized with the provided string.
     """
-    var bytes_buffer = List[Byte](s.as_bytes())
-    return Buffer(bytes_buffer^)
+    return Buffer(s.as_bytes())

gojo/bytes/buffer.mojo

@@ -56,6 +56,11 @@ struct Reader(
         slice."""
         return self.size - int(self.index)
 
+    @always_inline
+    fn __del__(owned self):
+        if self.data:
+            self.data.free()
+
     @always_inline
     fn as_bytes_slice(self: Reference[Self]) -> Span[UInt8, self.is_mutable, self.lifetime]:
         """Returns the internal data as a Span[UInt8]."""

gojo/bytes/reader.mojo

@@ -1,5 +1,4 @@
-from collections.optional import Optional
-from ..builtins import cap, copy, Byte, panic
+from ..builtins import copy, Byte, panic
 from .traits import ERR_UNEXPECTED_EOF
 
 alias BUFFER_SIZE = 4096