From c3401e60af5a23483432d983335029a41f94d5c9 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 2 Oct 2018 10:28:57 +0200 Subject: [PATCH 1/4] encode headings in csv writer --- commcare_export/writers.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/commcare_export/writers.py b/commcare_export/writers.py index ab5966c8..a8419d19 100644 --- a/commcare_export/writers.py +++ b/commcare_export/writers.py @@ -82,12 +82,17 @@ def write_table(self, table): if self.archive is None: raise Exception('Attempt to write to a closed CsvWriter') + def _encode_row(row): + return [ + val.encode('utf-8') if isinstance(val, six.text_type) else val + for val in row + ] + tempfile = StringIO() writer = csv.writer(tempfile, dialect=csv.excel) - writer.writerow(table['headings']) + writer.writerow(_encode_row(table['headings'])) for row in table['rows']: - writer.writerow([val.encode('utf-8') if isinstance(val, six.text_type) else val - for val in row]) + writer.writerow(_encode_row(row)) # TODO: make this a polite zip and put everything in a subfolder with the same basename # as the zipfile From b4a810509238cef7e4e0c4beba9072fdce45b1b0 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 2 Oct 2018 10:39:00 +0200 Subject: [PATCH 2/4] add test --- tests/test_writers.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_writers.py b/tests/test_writers.py index 22b41f9a..370c0ba9 100644 --- a/tests/test_writers.py +++ b/tests/test_writers.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function, absolute_import, division, generators, nested_scopes +import csv import datetime import tempfile +import zipfile import openpyxl import pytest import sqlalchemy -from commcare_export.writers import SqlTableWriter, JValueTableWriter, Excel2007TableWriter +from commcare_export.writers import SqlTableWriter, JValueTableWriter, Excel2007TableWriter, CsvTableWriter @pytest.fixture() @@ -100,6 +102,27 @@ def _check_Excel2007TableWriter_output(self, filename): ['4', '日本', '6'], ] + def test_CsvTableWriter(self): + with tempfile.NamedTemporaryFile() as file: + with CsvTableWriter(file=file) as writer: + writer.write_table({ + 'name': 'foo', + 'headings': ['a', 'bjørn', 'c'], + 'rows': [ + [1, '2', 3], + [4, '日本', 6], + ] + }) + + output_zip = zipfile.ZipFile(file.name, 'r') + output = csv.reader(output_zip.read('foo.csv').splitlines()) + + assert [[val.decode('utf-8') for val in row] for row in output] == [ + ['a', 'bjørn', 'c'], + ['1', '2', '3'], + ['4', '日本', '6'], + ] + @pytest.mark.dbtest class TestSQLWriters(object): From 9f8d603f3c99d1d5ff81c1caa329920692a393b8 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 2 Oct 2018 12:06:50 +0200 Subject: [PATCH 3/4] py3 + 2 compat --- commcare_export/writers.py | 12 ++++++------ setup.py | 3 ++- tests/test_cli.py | 2 +- tests/test_writers.py | 19 +++++++++++-------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/commcare_export/writers.py b/commcare_export/writers.py index a8419d19..d624f699 100644 --- a/commcare_export/writers.py +++ b/commcare_export/writers.py @@ -1,14 +1,14 @@ -import csv import datetime +import io import logging import sys import zipfile -from itertools import chain import alembic +import csv342 as csv import six import sqlalchemy -from six import StringIO, u +from six import u logger = logging.getLogger(__name__) @@ -84,11 +84,11 @@ def write_table(self, table): def _encode_row(row): return [ - val.encode('utf-8') if isinstance(val, six.text_type) else val + val.encode('utf-8') if isinstance(val, bytes) else val for val in row ] - tempfile = StringIO() + tempfile = io.StringIO() writer = csv.writer(tempfile, dialect=csv.excel) writer.writerow(_encode_row(table['headings'])) for row in table['rows']: @@ -97,7 +97,7 @@ def _encode_row(row): # TODO: make this a polite zip and put everything in a subfolder with the same basename # as the zipfile self.archive.writestr('%s.csv' % self.zip_safe_name(table['name']), - tempfile.getvalue()) + tempfile.getvalue().encode('utf-8')) def __exit__(self, exc_type, exc_val, exc_tb): self.archive.close() diff --git a/setup.py b/setup.py index 506fcafa..e7fa6a33 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,8 @@ def run_tests(self): 'sqlalchemy', 'pytz', 'sqlalchemy-migrate', - 'backoff' + 'backoff', + 'csv342' ], tests_require = ['pytest', 'psycopg2', 'mock'], cmdclass = {'test': PyTest}, diff --git a/tests/test_cli.py b/tests/test_cli.py index 7e7d24e0..76752f11 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import csv +import csv342 as csv import os import unittest from argparse import Namespace diff --git a/tests/test_writers.py b/tests/test_writers.py index 370c0ba9..ea4eab62 100644 --- a/tests/test_writers.py +++ b/tests/test_writers.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function, absolute_import, division, generators, nested_scopes -import csv +import csv342 as csv import datetime +import io import tempfile import zipfile @@ -114,14 +115,16 @@ def test_CsvTableWriter(self): ] }) - output_zip = zipfile.ZipFile(file.name, 'r') - output = csv.reader(output_zip.read('foo.csv').splitlines()) + with zipfile.ZipFile(file.name, 'r') as output_zip: + output_zip.extract('foo.csv', '/home/skelly/foo.csv') + with output_zip.open('foo.csv') as csv_file: + output = csv.reader(io.TextIOWrapper(csv_file, encoding='utf-8')) - assert [[val.decode('utf-8') for val in row] for row in output] == [ - ['a', 'bjørn', 'c'], - ['1', '2', '3'], - ['4', '日本', '6'], - ] + assert [row for row in output] == [ + ['a', 'bjørn', 'c'], + ['1', '2', '3'], + ['4', '日本', '6'], + ] @pytest.mark.dbtest From ecd0a336650e4dd858be9d7dce5b65e0e253001f Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Mon, 8 Oct 2018 11:20:57 +0200 Subject: [PATCH 4/4] remove test line --- tests/test_writers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_writers.py b/tests/test_writers.py index ea4eab62..f25c94b2 100644 --- a/tests/test_writers.py +++ b/tests/test_writers.py @@ -116,7 +116,6 @@ def test_CsvTableWriter(self): }) with zipfile.ZipFile(file.name, 'r') as output_zip: - output_zip.extract('foo.csv', '/home/skelly/foo.csv') with output_zip.open('foo.csv') as csv_file: output = csv.reader(io.TextIOWrapper(csv_file, encoding='utf-8'))