Better unicode support

.config/corpus_report.markdownlint.yaml

@@ -0,0 +1,3 @@
+# Markdownlint configuration for corpus test reports
+# Disable line length rule since tables naturally exceed 80 characters
+MD013: false

.github/github_sucks

@@ -0,0 +1,3 @@
+
+
+

.github/workflows/test.yaml

@@ -30,3 +30,47 @@ jobs:
           image: danielflook/python-minifier-build:${{ matrix.python }}-2024-09-15
           run: |
             tox -r -e $(echo "${{ matrix.python }}" | tr -d .)
+
+  test-windows:
+    name: Test Windows
+    runs-on: windows-2022
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.2.2
+        with:
+          fetch-depth: 1
+          show-progress: false
+          persist-credentials: false
+
+      - name: Set up Python
+        if: ${{ matrix.python-version != '2.7' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up Python
+        if: ${{ matrix.python-version == '2.7' }}
+        uses: LizardByte/actions/actions/setup_python@eddc8fc8b27048e25040e37e3585bd3ef9a968ed  # master
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set version statically
+        shell: powershell
+        run: |
+          $content = Get-Content setup.py
+          $content = $content -replace "setup_requires=.*", "version='0.0.0',"
+          $content = $content -replace "use_scm_version=.*", ""
+          Set-Content setup.py $content
+
+      - name: Install tox
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Run tests
+        run: |
+          tox -c tox-windows.ini -r -e ${{ matrix.python-version }}

.github/workflows/test_corpus.yaml

@@ -144,4 +144,11 @@ jobs:
           volumes: |
             /corpus-results:/corpus-results
           run: |
-            python3.13 workflow/corpus_test/generate_report.py /corpus-results ${{ inputs.ref }} ${{ steps.ref.outputs.commit }} ${{ inputs.base-ref }} ${{ steps.base-ref.outputs.commit }} >> $GITHUB_STEP_SUMMARY
+            python3.13 workflow/corpus_test/generate_report.py /corpus-results ${{ inputs.ref }} ${{ steps.ref.outputs.commit }} ${{ inputs.base-ref }} ${{ steps.base-ref.outputs.commit }} | tee -a $GITHUB_STEP_SUMMARY > report.md
+
+      - name: Lint Report
+        uses: DavidAnson/markdownlint-cli2-action@05f32210e84442804257b2a6f20b273450ec8265 # v19
+        continue-on-error: true
+        with:
+          config: '.config/corpus_report.markdownlint.yaml'
+          globs: 'report.md'

corpus_test/generate_report.py

@@ -9,6 +9,33 @@
 
 ENHANCED_REPORT = os.environ.get('ENHANCED_REPORT', True)
 
+def is_recursion_error(python_version: str, result: Result) -> bool:
+    """
+    Check if the result is a recursion error
+    """
+    if result.outcome == 'RecursionError':
+        return True
+
+    if python_version in ['2.7', '3.3', '3.4']:
+        # In these versions, the recursion error is raised as an Exception
+        return result.outcome.startswith('Exception: maximum recursion depth exceeded')
+
+    return False
+
+def is_syntax_error(python_version: str, result: Result) -> bool:
+    """
+    Check if the result is a syntax error
+    """
+    if result.outcome == 'SyntaxError':
+        return True
+
+    if python_version == '2.7' and result.outcome == 'Exception: compile() expected string without null bytes':
+        return True
+
+    if python_version != '2.7' and result.outcome == 'Exception: source code string cannot contain null bytes':
+        return True
+
+    return False
 
 @dataclass
 class ResultSet:
@@ -45,11 +72,11 @@ def add(self, result: Result):
             if result.original_size < result.minified_size:
                 self.larger_than_original_count += 1
 
-        if result.outcome == 'RecursionError':
+        if is_recursion_error(self.python_version, result):
             self.recursion_error_count += 1
         elif result.outcome == 'UnstableMinification':
             self.unstable_minification_count += 1
-        elif result.outcome.startswith('Exception'):
+        elif result.outcome.startswith('Exception') and not is_syntax_error(self.python_version, result):
             self.exception_count += 1
 
     @property
@@ -74,13 +101,13 @@ def larger_than_original(self) -> Iterable[Result]:
     def recursion_error(self) -> Iterable[Result]:
         """Return those entries that have a recursion error"""
         for result in self.entries.values():
-            if result.outcome == 'RecursionError':
+            if is_recursion_error(self.python_version, result):
                 yield result
 
     def exception(self) -> Iterable[Result]:
         """Return those entries that have an exception"""
         for result in self.entries.values():
-            if result.outcome.startswith('Exception'):
+            if result.outcome.startswith('Exception') and not is_syntax_error(self.python_version, result) and not is_recursion_error(self.python_version, result):
                 yield result
 
     def unstable_minification(self) -> Iterable[Result]:
@@ -184,7 +211,7 @@ def format_difference(compare: Iterable[Result], base: Iterable[Result]) -> str:
         return s
 
 
-def report_larger_than_original(results_dir: str, python_versions: str, minifier_sha: str) -> str:
+def report_larger_than_original(results_dir: str, python_versions: list[str], minifier_sha: str) -> str:
     yield '''
 ## Larger than original
 
@@ -203,7 +230,7 @@ def report_larger_than_original(results_dir: str, python_versions: str, minifier
             yield f'| {entry.corpus_entry} | {entry.original_size} | {entry.minified_size} ({entry.minified_size - entry.original_size:+}) |'
 
 
-def report_unstable(results_dir: str, python_versions: str, minifier_sha: str) -> str:
+def report_unstable(results_dir: str, python_versions: list[str], minifier_sha: str) -> str:
     yield '''
 ## Unstable
 
@@ -222,7 +249,7 @@ def report_unstable(results_dir: str, python_versions: str, minifier_sha: str) -
             yield f'| {entry.corpus_entry} | {python_version} | {entry.original_size} |'
 
 
-def report_exceptions(results_dir: str, python_versions: str, minifier_sha: str) -> str:
+def report_exceptions(results_dir: str, python_versions: list[str], minifier_sha: str) -> str:
     yield '''
 ## Exceptions
 
@@ -244,10 +271,10 @@ def report_exceptions(results_dir: str, python_versions: str, minifier_sha: str)
             yield f'| {entry.corpus_entry} | {python_version} | {entry.outcome} |'
 
     if not exceptions_found:
-        yield ' None | | |'
+        yield '| None | | |'
 
 
-def report_larger_than_base(results_dir: str, python_versions: str, minifier_sha: str, base_sha: str) -> str:
+def report_larger_than_base(results_dir: str, python_versions: list[str], minifier_sha: str, base_sha: str) -> str:
     yield '''
 ## Top 10 Larger than base
 
@@ -277,7 +304,7 @@ def report_larger_than_base(results_dir: str, python_versions: str, minifier_sha
         yield '| N/A | N/A | N/A |'
 
 
-def report_slowest(results_dir: str, python_versions: str, minifier_sha: str) -> str:
+def report_slowest(results_dir: str, python_versions: list[str], minifier_sha: str) -> str:
     yield '''
 ## Top 10 Slowest
 
@@ -360,15 +387,15 @@ def report(results_dir: str, minifier_ref: str, minifier_sha: str, base_ref: str
                 f'| {format_difference(summary.larger_than_original(), base_summary.larger_than_original())} ' +
                 f'| {format_difference(summary.recursion_error(), base_summary.recursion_error())} ' +
                 f'| {format_difference(summary.unstable_minification(), base_summary.unstable_minification())} ' +
-                f'| {format_difference(summary.exception(), base_summary.exception())} '
+                f'| {format_difference(summary.exception(), base_summary.exception())} |'
         )
 
     if ENHANCED_REPORT:
         yield from report_larger_than_original(results_dir, ['3.13'], minifier_sha)
         yield from report_larger_than_base(results_dir, ['3.13'], minifier_sha, base_sha)
         yield from report_slowest(results_dir, ['3.13'], minifier_sha)
         yield from report_unstable(results_dir, ['2.7', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'], minifier_sha)
-        yield from report_exceptions(results_dir, ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'], minifier_sha)
+        yield from report_exceptions(results_dir, ['2.7', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'], minifier_sha)
 
 
 def main():

corpus_test/generate_results.py

@@ -50,6 +50,10 @@ def minify_corpus_entry(corpus_path, corpus_entry):
         # Source is too deep
         result.outcome = 'RecursionError'
 
+    except ValueError:
+        # Source is not valid Python
+        result.outcome = 'ValueError'
+
     except SyntaxError:
         # Source not valid for this version of Python
         result.outcome = 'SyntaxError'

src/python_minifier/__init__.pyi

@@ -1,6 +1,6 @@
 import ast
 
-from typing import Any, AnyStr, List, Optional, Text, Union
+from typing import Any, List, Optional, Text, Union
 
 from .transforms.remove_annotations_options import RemoveAnnotationsOptions as RemoveAnnotationsOptions
 
@@ -10,7 +10,7 @@ class UnstableMinification(RuntimeError):
 
 
 def minify(
-    source: AnyStr,
+    source: Union[str, bytes],
     filename: Optional[str] = ...,
     remove_annotations: Union[bool, RemoveAnnotationsOptions] = ...,
     remove_pass: bool = ...,
@@ -36,7 +36,7 @@ def unparse(module: ast.Module) -> Text: ...
 
 
 def awslambda(
-    source: AnyStr,
+    source: Union[str, bytes],
     filename: Optional[Text] = ...,
     entrypoint: Optional[Text] = ...
 ) -> Text: ...

src/python_minifier/__main__.py

@@ -7,6 +7,27 @@
 from python_minifier import minify
 from python_minifier.transforms.remove_annotations_options import RemoveAnnotationsOptions
 
+# Python 2.7 compatibility for UTF-8 file writing
+if sys.version_info[0] == 2:
+    import codecs
+    def open_utf8(filename, mode):
+        return codecs.open(filename, mode, encoding='utf-8')
+else:
+    def open_utf8(filename, mode):
+        return open(filename, mode, encoding='utf-8')
+
+def safe_stdout_write(text):
+    """Write text to stdout with proper encoding handling."""
+    try:
+        sys.stdout.write(text)
+    except UnicodeEncodeError:
+        # Fallback: encode to UTF-8 and write to stdout.buffer (Python 3) or sys.stdout (Python 2)
+        if sys.version_info[0] >= 3 and hasattr(sys.stdout, 'buffer'):
+            sys.stdout.buffer.write(text.encode('utf-8'))
+        else:
+            # Python 2.7 or no buffer attribute - write UTF-8 encoded bytes
+            sys.stdout.write(text.encode('utf-8'))
+
 
 if sys.version_info >= (3, 8):
     from importlib import metadata
@@ -53,10 +74,10 @@ def main():
         source = sys.stdin.buffer.read() if sys.version_info >= (3, 0) else sys.stdin.read()
         minified = do_minify(source, 'stdin', args)
         if args.output:
-            with open(args.output, 'w') as f:
+            with open_utf8(args.output, 'w') as f:
                 f.write(minified)
         else:
-            sys.stdout.write(minified)
+            safe_stdout_write(minified)
 
     else:
         # minify source paths
@@ -70,13 +91,13 @@ def main():
             minified = do_minify(source, path, args)
 
             if args.in_place:
-                with open(path, 'w') as f:
+                with open_utf8(path, 'w') as f:
                     f.write(minified)
             elif args.output:
-                with open(args.output, 'w') as f:
+                with open_utf8(args.output, 'w') as f:
                     f.write(minified)
             else:
-                sys.stdout.write(minified)
+                safe_stdout_write(minified)
 
 
 def parse_args():

src/python_minifier/module_printer.py

@@ -28,11 +28,15 @@ def __call__(self, module):
         assert isinstance(module, ast.Module)
 
         self.visit_Module(module)
-        return str(self.printer).rstrip('\n' + self.indent_char + ';')
+        # On Python 2.7, preserve unicode strings to avoid encoding issues
+        code = unicode(self.printer) if sys.version_info[0] < 3 else str(self.printer)
+        return code.rstrip('\n' + self.indent_char + ';')
 
     @property
     def code(self):
-        return str(self.printer).rstrip('\n' + self.indent_char + ';')
+        # On Python 2.7, preserve unicode strings to avoid encoding issues
+        code = unicode(self.printer) if sys.version_info[0] < 3 else str(self.printer)
+        return code.rstrip('\n' + self.indent_char + ';')
 
     # region Simple Statements
 

src/python_minifier/token_printer.py

@@ -91,14 +91,22 @@ def __init__(self, prefer_single_line=False, allow_invalid_num_warnings=False):
         self._prefer_single_line = prefer_single_line
         self._allow_invalid_num_warnings = allow_invalid_num_warnings
 
-        self._code = ''
+        # Initialize as unicode string on Python 2.7 to handle Unicode content
+        if sys.version_info[0] < 3:
+            self._code = u''
+        else:
+            self._code = ''
         self.indent = 0
         self.unicode_literals = False
         self.previous_token = TokenTypes.NoToken
 
     def __str__(self):
         """Return the output code."""
         return self._code
+
+    def __unicode__(self):
+        """Return the output code as unicode (for Python 2.7 compatibility)."""
+        return self._code
 
     def identifier(self, name):
         """Add an identifier to the output code."""

test/test_is_constant_node.py

@@ -7,6 +7,11 @@
 from python_minifier.util import is_constant_node
 
 
+@pytest.mark.filterwarnings("ignore:ast.Str is deprecated:DeprecationWarning")
+@pytest.mark.filterwarnings("ignore:ast.Bytes is deprecated:DeprecationWarning") 
+@pytest.mark.filterwarnings("ignore:ast.Num is deprecated:DeprecationWarning")
+@pytest.mark.filterwarnings("ignore:ast.NameConstant is deprecated:DeprecationWarning")
+@pytest.mark.filterwarnings("ignore:ast.Ellipsis is deprecated:DeprecationWarning")
 def test_type_nodes():
     assert is_constant_node(ast.Str('a'), ast.Str)
 
@@ -28,6 +33,11 @@ def test_type_nodes():
     assert is_constant_node(ast.Ellipsis(), ast.Ellipsis)
 
 
+@pytest.mark.filterwarnings("ignore:ast.Str is deprecated:DeprecationWarning")
+@pytest.mark.filterwarnings("ignore:ast.Bytes is deprecated:DeprecationWarning") 
+@pytest.mark.filterwarnings("ignore:ast.Num is deprecated:DeprecationWarning")
+@pytest.mark.filterwarnings("ignore:ast.NameConstant is deprecated:DeprecationWarning")
+@pytest.mark.filterwarnings("ignore:ast.Ellipsis is deprecated:DeprecationWarning")
 def test_constant_nodes():
     # only test on python 3.8+
     if sys.version_info < (3, 8):