feature/python3.11 (#22)

* now works with python3.11 and automatically converts data to utf8

* updated README

* updated poetry.lock

---------

Co-authored-by: george <g.e.c.burton@gmal.com>

Author: gecBurton

README.md

@@ -1,13 +1,13 @@
 # clean your CSVs!
 
 This command line tool cleans CSV files by:
-1. detecting the encoding and converting it to utf-8
+1. converting encoding to utf-8
 2. detecting the delimiter and safely converting it to a comma
 3. casting all variables to json form, i.e. integers, floats, booleans, string or null.
 
 
 * install `pip install csv-bleach`
-* and run like `python -m run csv_bleach my-data.csv`
+* and run like `python -m csv_bleach my-data.csv`
 
 The only option is the output file name, by default it will be your original file name with `.scsv` extension.
 

csv_bleach/__main__.py

@@ -4,7 +4,6 @@
 
 import click
 
-from csv_bleach.detect_encoding import detect_encoding
 from csv_bleach.detect_row_count import detect_row_count
 from csv_bleach.type_casting import infer_types
 
@@ -24,15 +23,12 @@ def cli(file: str, output: Optional[str]):
         output = f"{filepath}.scsv"
 
     with open(file, "rb") as input_file:
-        encoding = detect_encoding(input_file)
-
-    with open(file, encoding=encoding) as input_file:
         row_count = detect_row_count(input_file)
 
-    with open(file, encoding=encoding) as input_file:
+    with open(file, "rb") as input_file:
         type_caster = infer_types(input_file)
 
-    with open(file, encoding=encoding) as input_file, open(output, "w") as output_file:
+    with open(file, "rb") as input_file, open(output, "w") as output_file:
         type_caster.process_file(input_file, output_file, row_count)
 
 

csv_bleach/detect_delimiter.py

@@ -6,7 +6,7 @@
 
 QUOTE = 34
 NEW_LINE = 10
-
+CARRIAGE_RETURN = 13
 
 class DelimiterDetector:
     def __init__(self, delimiter_count: dict[str, int]):
@@ -25,7 +25,7 @@ def parse_row(cls, txt: str) -> DelimiterDetector:
                 not escaped
                 and not char.isalnum()
                 and not (char.isspace() and prev and prev.isspace())
-                and ord(char) not in (NEW_LINE, QUOTE)
+                and ord(char) not in (NEW_LINE, QUOTE, CARRIAGE_RETURN)
             ):
                 chars.append(char)
 

csv_bleach/detect_encoding.py

@@ -1,13 +1,5 @@
-from typing import IO
+from typing import BinaryIO
 
 
-def blocks(files, size=65536):
-    while True:
-        b = files.read(size)
-        if not b:
-            break
-        yield b
-
-
-def detect_row_count(file: IO[str]) -> int:
-    return sum(bl.count("\n") for bl in blocks(file))
+def detect_row_count(file: BinaryIO) -> int:
+    return sum(1 for _ in file)

csv_bleach/detect_row_count.py

@@ -1,9 +1,10 @@
 import json
 import logging
-from typing import Any, Iterator, List, TextIO
+from typing import Any, Iterator, List, BinaryIO, TextIO
 
 import click
 
+from charset_normalizer import from_bytes
 from csv_bleach.detect_delimiter import DelimiterDetector
 from csv_bleach.line_decoder import LineSplit
 
@@ -53,13 +54,14 @@ def type_cast_row(self, i: int, txt: str) -> List[Any]:
 
         return words
 
-    def parse_file(self, rows: TextIO) -> Iterator[list]:
+    def parse_file(self, rows: BinaryIO) -> Iterator[list]:
         for i, row in enumerate(rows):
-            if len(row.strip()) > 0:
-                typed_row = self.type_cast_row(i, row)
+            str_row = str(from_bytes(row).best())
+            if len(str_row.strip()) > 0:
+                typed_row = self.type_cast_row(i, str_row)
                 yield typed_row
 
-    def process_file(self, input_file: TextIO, output_file: TextIO, row_count: int):
+    def process_file(self, input_file: BinaryIO, output_file: TextIO, row_count: int):
         with click.progressbar(
             self.parse_file(input_file),
             length=row_count,
@@ -70,11 +72,12 @@ def process_file(self, input_file: TextIO, output_file: TextIO, row_count: int):
                 output_file.write(json_row)
 
 
-def infer_types(rows: TextIO) -> TypeCaster:
+def infer_types(rows: BinaryIO) -> TypeCaster:
     def _read(_rows):
         for row in _rows:
-            if len(row.strip()) > 0:
-                yield DelimiterDetector.parse_row(row)
+            str_row = str(from_bytes(row).best())
+            if len(str_row.strip()) > 0:
+                yield DelimiterDetector.parse_row(str_row)
 
     dd = DelimiterDetector.combine(_read(rows))
     assert len(dd.delimiter_count) == 1, dd.delimiter_count