From b6c4a45d5f0f3a7e1e053ff0356483a45e5e1f4b Mon Sep 17 00:00:00 2001 From: Jim Nicholls Date: Sun, 6 Sep 2015 10:56:22 +1000 Subject: [PATCH 1/7] Add close to Writer. --- pymarc/writer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymarc/writer.py b/pymarc/writer.py index 8e29032..635df29 100644 --- a/pymarc/writer.py +++ b/pymarc/writer.py @@ -1,10 +1,15 @@ from pymarc import Record, WriteNeedsRecord + class Writer(object): def write(self, record): pass + def close(self): + pass + + class MARCWriter(Writer): """ A class for writing MARC21 records in transmission format. From 2c502707b3ae74ad2dc8c601df256a9e650f6fd7 Mon Sep 17 00:00:00 2001 From: Jim Nicholls Date: Sun, 6 Sep 2015 11:45:50 +1000 Subject: [PATCH 2/7] Add own_file_handle to MARCWriter to control if close also closes the file handle. --- pymarc/writer.py | 12 ++++++++++-- test/writer.py | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/pymarc/writer.py b/pymarc/writer.py index 635df29..acab7bd 100644 --- a/pymarc/writer.py +++ b/pymarc/writer.py @@ -29,12 +29,17 @@ class MARCWriter(Writer): print string """ - def __init__(self, file_handle): + def __init__(self, file_handle, own_file_handle = True): """ You need to pass in a file like object. + + If own_file_handle is True (the default) then the file handle will be + closed when the writer is closed. Otherwise the file handle will be + left open. """ super(MARCWriter, self).__init__() self.file_handle = file_handle + self.own_file_handle = own_file_handle def write(self, record): """ @@ -47,7 +52,10 @@ def write(self, record): def close(self): """ Closes the file. + + If own_file_handle is True, also closes the file handle. """ - self.file_handle.close() + if self.own_file_handle: + self.file_handle.close() self.file_handle = None diff --git a/test/writer.py b/test/writer.py index 782415a..d2778cd 100644 --- a/test/writer.py +++ b/test/writer.py @@ -1,18 +1,24 @@ import unittest import pymarc import os +from six import BytesIO + class MARCWriterTest(unittest.TestCase): def test_write(self): # write a record off to a file - writer = pymarc.MARCWriter(open('test/writer-test.dat', 'wb')) + file_handle = open('test/writer-test.dat', 'wb') + writer = pymarc.MARCWriter(file_handle) record = pymarc.Record() field = pymarc.Field('245', ['0', '0'], ['a', 'foo']) record.add_field(field) writer.write(record) writer.close() + self.assertTrue( + file_handle.closed, + 'The file handle should close when the writer closes') # read it back in reader = pymarc.MARCReader(open('test/writer-test.dat', 'rb')) @@ -22,6 +28,42 @@ def test_write(self): # remove it os.remove('test/writer-test.dat') + def test_own_file_handle_true(self): + """ + If a MARCWriter is created with own_file_handle = True, then when the + MARCWriter is closed the file handle is also closed. + """ + file_handle = BytesIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.MARCWriter(file_handle, own_file_handle = True) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertTrue( + file_handle.closed, + 'The file handle should close when the writer closes') + + def test_own_file_handle_false(self): + """ + If a MARCWriter is created with own_file_handle = False, then when the + MARCWriter is closed the file handle is NOT also closed. + """ + file_handle = BytesIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.MARCWriter(file_handle, own_file_handle = False) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertFalse( + file_handle.closed, + 'The file handle should NOT close when the writer closes') + def suite(): test_suite = unittest.makeSuite(MARCWriterTest, 'test') return test_suite From 2a392b9c840c6c2f3be45080e6dc9fc2d5ddfdb2 Mon Sep 17 00:00:00 2001 From: Jim Nicholls Date: Sun, 6 Sep 2015 13:18:24 +1000 Subject: [PATCH 3/7] Implement TextWriter to write files of records in text format. --- pymarc/writer.py | 57 ++++++++++++++++++ test/writer.py | 149 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 2 deletions(-) diff --git a/pymarc/writer.py b/pymarc/writer.py index acab7bd..0dd06bb 100644 --- a/pymarc/writer.py +++ b/pymarc/writer.py @@ -59,3 +59,60 @@ def close(self): self.file_handle.close() self.file_handle = None + +class TextWriter(Writer): + """ + A class for writing records in prettified text MARCMaker format. + + A blank line separates each record. + + Simple usage: + + from pymarc import TextWriter + + ## pass in a file + writer = TextWriter(open('file.dat','wt')) + writer.write(record) + writer.close() + + ## use StringIO if you want to write to a string + string = StringIO() + writer = MARCWriter(string, own_file_handle = False) + writer.write(record) + writer.close() + print string + """ + + def __init__(self, file_handle, own_file_handle = True): + """ + You need to pass in a file like object. + + If own_file_handle is True (the default) then the file handle will be + closed when the writer is closed. Otherwise the file handle will be + left open. + """ + super(TextWriter, self).__init__() + self.file_handle = file_handle + self.own_file_handle = own_file_handle + self.write_count = 0 + + def write(self, record): + """ + Writes a record. + """ + if not isinstance(record, Record): + raise WriteNeedsRecord + if self.write_count > 0: + self.file_handle.write('\n') + self.file_handle.write(str(record)) + self.write_count += 1 + + def close(self): + """ + Closes the file. + + If own_file_handle is True, also closes the file handle. + """ + if self.own_file_handle: + self.file_handle.close() + self.file_handle = None diff --git a/test/writer.py b/test/writer.py index d2778cd..7d18395 100644 --- a/test/writer.py +++ b/test/writer.py @@ -1,7 +1,8 @@ import unittest import pymarc import os -from six import BytesIO +import textwrap +from six import BytesIO, StringIO, u class MARCWriterTest(unittest.TestCase): @@ -64,8 +65,152 @@ def test_own_file_handle_false(self): file_handle.closed, 'The file handle should NOT close when the writer closes') + +class TextWriterTest(unittest.TestCase): + + def test_writing_0_records(self): + file_handle = StringIO() + try: + writer = pymarc.TextWriter(file_handle, own_file_handle = False) + writer.close() + self.assertEqual( + file_handle.getvalue(), + '', + 'Nothing should be have been written to the file handle') + finally: + file_handle.close() + + def test_writing_1_record(self): + expected = r""" + =LDR 22 4500 + =100 00$ame + =245 00$aFoo /$cby me. + """ + expected = textwrap.dedent(expected[1:]) + file_handle = StringIO() + try: + writer = pymarc.TextWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + writer.close() + self.assertEquals(file_handle.getvalue(), expected) + finally: + file_handle.close() + + def test_writing_3_records(self): + expected = r""" + =LDR 22 4500 + =008 090227s2009\\\\mau\\\\\\\\\\\\\\\\\chi\d + =100 00$ame + =245 00$aFoo /$cby me. + + =LDR 22 4500 + =100 00$ame + =245 00$aFoo /$cby me. + + =LDR 22 4500 + =245 00$aFoo /$cby me. + """ + expected = textwrap.dedent(expected[1:]) + file_handle = StringIO() + try: + writer = pymarc.TextWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + record.add_field( + pymarc.Field( + '008', + data=u('090227s2009 mau chi d'))) + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + record = pymarc.Record() + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + record = pymarc.Record() + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + writer.close() + self.assertEquals(file_handle.getvalue(), expected) + finally: + file_handle.close() + + def test_writing_empty_record(self): + expected = r""" + =LDR 22 4500 + """ + expected = textwrap.dedent(expected[1:]) + file_handle = StringIO() + try: + writer = pymarc.TextWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + writer.write(record) + self.assertEquals(file_handle.getvalue(), expected) + finally: + file_handle.close() + + def test_own_file_handle_true(self): + """ + If a TextWriter is created with own_file_handle = True, then when the + TextWriter is closed the file handle is also closed. + """ + file_handle = StringIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.TextWriter(file_handle, own_file_handle = True) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertTrue( + file_handle.closed, + 'The file handle should close when the writer closes') + + def test_own_file_handle_false(self): + """ + If a TextWriter is created with own_file_handle = False, then when the + TextWriter is closed the file handle is NOT also closed. + """ + file_handle = StringIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.TextWriter(file_handle, own_file_handle = False) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertFalse( + file_handle.closed, + 'The file handle should NOT close when the writer closes') + + def suite(): - test_suite = unittest.makeSuite(MARCWriterTest, 'test') + marc_suite = unittest.makeSuite(MARCWriterTest, 'test') + text_suite = unittest.makeSuite(TextWriterTest, 'test') + test_suite = unittest.TestSuite((marc_suite, text_suite)) return test_suite if __name__ == '__main__': From 780ff5ae2952f43ec246d01e7c91b64a831de3a7 Mon Sep 17 00:00:00 2001 From: Jim Nicholls Date: Sun, 6 Sep 2015 14:48:06 +1000 Subject: [PATCH 4/7] Implement XMLWriter to write files of records as a MARCXML collection. --- pymarc/writer.py | 68 +++++++++++++++++ test/writer.py | 193 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 259 insertions(+), 2 deletions(-) diff --git a/pymarc/writer.py b/pymarc/writer.py index 0dd06bb..0546fc8 100644 --- a/pymarc/writer.py +++ b/pymarc/writer.py @@ -1,5 +1,11 @@ +import pymarc from pymarc import Record, WriteNeedsRecord +try: + import xml.etree.ElementTree as ET # builtin in Python 2.5 +except ImportError: + import elementtree.ElementTree as ET + class Writer(object): @@ -116,3 +122,65 @@ def close(self): if self.own_file_handle: self.file_handle.close() self.file_handle = None + + +class XMLWriter(Writer): + """ + A class for writing records as a MARCXML collection. + + IMPORTANT: You must the close an XMLWriter, otherwise you will not get + a valid XML document. + + Simple usage: + + from pymarc import XMLWriter + + ## pass in a file + writer = XMLWriter(open('file.xml','wb')) + writer.write(record) + writer.close() + + ## use StringIO for Python 2 and BytesIO for Python 3 + ## if you want to write to a string/bytes. + string = StringIO() ## BytesIO() for Python 3 + writer = MARCWriter(string, own_file_handle = False) + writer.write(record) + writer.close() + print string + """ + + def __init__(self, file_handle, own_file_handle = True): + """ + You need to pass in a binary file like object. + + If own_file_handle is True (the default) then the file handle will be + closed when the writer is closed. Otherwise the file handle will be + left open. + """ + super(XMLWriter, self).__init__() + self.file_handle = file_handle + self.own_file_handle = own_file_handle + self.file_handle.write( + b'') + self.file_handle.write( + b'') + + def write(self, record): + """ + Writes a record. + """ + if not isinstance(record, Record): + raise WriteNeedsRecord + node = pymarc.record_to_xml_node(record) + self.file_handle.write(ET.tostring(node, encoding='utf-8')) + + def close(self): + """ + Closes the file. + + If own_file_handle is True, also closes the file handle. + """ + self.file_handle.write(b'') + if self.own_file_handle: + self.file_handle.close() + self.file_handle = None diff --git a/test/writer.py b/test/writer.py index 7d18395..510d73a 100644 --- a/test/writer.py +++ b/test/writer.py @@ -2,7 +2,7 @@ import pymarc import os import textwrap -from six import BytesIO, StringIO, u +from six import BytesIO, StringIO, u, binary_type class MARCWriterTest(unittest.TestCase): @@ -207,10 +207,199 @@ def test_own_file_handle_false(self): 'The file handle should NOT close when the writer closes') +class XMLWriterTest(unittest.TestCase): + + def test_writing_0_records(self): + expected = r""" + + + + """ + expected = textwrap.dedent(expected[1:]).replace('\n', '') + if str != binary_type: + expected = expected.encode() + file_handle = BytesIO() + try: + writer = pymarc.XMLWriter(file_handle, own_file_handle = False) + writer.close() + self.assertEquals(file_handle.getvalue(), expected) + finally: + file_handle.close() + + def test_writing_empty_record(self): + expected = r""" + + + + 22 4500 + + + """ + expected = textwrap.dedent(expected[1:]).replace('\n', '') + if str != binary_type: + expected = expected.encode() + file_handle = BytesIO() + try: + writer = pymarc.XMLWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + writer.write(record) + writer.close() + self.assertEquals(file_handle.getvalue(), expected) + finally: + file_handle.close() + + def test_writing_1_record(self): + expected = r""" + + + + 22 4500 + + me + + + Foo / + by me. + + + + """ + expected = textwrap.dedent(expected[1:]).replace('\n', '') + if str != binary_type: + expected = expected.encode() + file_handle = BytesIO() + try: + writer = pymarc.XMLWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + writer.close() + self.assertEquals(file_handle.getvalue(), expected) + finally: + file_handle.close() + + def test_writing_3_records(self): + expected = r""" + + + + 22 4500 + 090227s2009 mau chi d + + me + + + Foo / + by me. + + + + 22 4500 + + me + + + Foo / + by me. + + + + 22 4500 + + Foo / + by me. + + + + """ + expected = textwrap.dedent(expected[1:]).replace('\n', '') + if str != binary_type: + expected = expected.encode() + file_handle = BytesIO() + try: + writer = pymarc.XMLWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + record.add_field( + pymarc.Field( + '008', + data=u('090227s2009 mau chi d'))) + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + record = pymarc.Record() + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + record = pymarc.Record() + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + writer.close() + self.assertEquals(file_handle.getvalue(), expected) + finally: + file_handle.close() + + def test_own_file_handle_true(self): + """ + If a XMLWriter is created with own_file_handle = True, then when the + XMLWriter is closed the file handle is also closed. + """ + file_handle = BytesIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.XMLWriter(file_handle, own_file_handle = True) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertTrue( + file_handle.closed, + 'The file handle should close when the writer closes') + + def test_own_file_handle_false(self): + """ + If a XMLWriter is created with own_file_handle = False, then when the + XMLWriter is closed the file handle is NOT also closed. + """ + file_handle = BytesIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.XMLWriter(file_handle, own_file_handle = False) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertFalse( + file_handle.closed, + 'The file handle should NOT close when the writer closes') + + def suite(): marc_suite = unittest.makeSuite(MARCWriterTest, 'test') text_suite = unittest.makeSuite(TextWriterTest, 'test') - test_suite = unittest.TestSuite((marc_suite, text_suite)) + xml_suite = unittest.makeSuite(XMLWriterTest, 'test') + test_suite = unittest.TestSuite((marc_suite, text_suite, xml_suite)) return test_suite if __name__ == '__main__': From 699e63a2f00e3ed8514726eaac89455d98d77545 Mon Sep 17 00:00:00 2001 From: Jim Nicholls Date: Sun, 6 Sep 2015 15:52:58 +1000 Subject: [PATCH 5/7] Implement JSONWriter to write files of records as an array of MARC-in-JSON objects. --- pymarc/writer.py | 72 ++++++++++++++ test/writer.py | 242 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 313 insertions(+), 1 deletion(-) diff --git a/pymarc/writer.py b/pymarc/writer.py index 0546fc8..bd6640d 100644 --- a/pymarc/writer.py +++ b/pymarc/writer.py @@ -6,6 +6,17 @@ except ImportError: import elementtree.ElementTree as ET +try: + # the json module was included in the stdlib in python 2.6 + # http://docs.python.org/library/json.html + import json +except ImportError: + # simplejson 2.0.9 is available for python 2.4+ + # http://pypi.python.org/pypi/simplejson/2.0.9 + # simplejson 1.7.3 is available for python 2.3+ + # http://pypi.python.org/pypi/simplejson/1.7.3 + import simplejson as json + class Writer(object): @@ -16,6 +27,67 @@ def close(self): pass +class JSONWriter(Writer): + """ + A class for writing records as an array of MARC-in-JSON objects. + + IMPORTANT: You must the close a JSONWriter, + otherwise you will not get valid JSON. + + Simple usage: + + from pymarc import JSONWriter + + ## pass in a file + writer = JSONWriter(open('file.xml','wt')) + writer.write(record) + writer.close() + + ## use StringIO if you want to write to a string + string = StringIO() + writer = JSONWriter(string, own_file_handle = False) + writer.write(record) + writer.close() + print string + """ + + def __init__(self, file_handle, own_file_handle = True): + """ + You need to pass in a text file like object. + + If own_file_handle is True (the default) then the file handle will be + closed when the writer is closed. Otherwise the file handle will be + left open. + """ + super(JSONWriter, self).__init__() + self.file_handle = file_handle + self.own_file_handle = own_file_handle + self.write_count = 0 + self.file_handle.write('[') + + def write(self, record): + """ + Writes a record. + """ + if not isinstance(record, Record): + raise WriteNeedsRecord + if self.write_count > 0: + self.file_handle.write(',') + json.dump(record.as_dict(), self.file_handle, separators=(',', ':')) + self.write_count += 1 + + def close(self): + """ + Closes the file. + + If own_file_handle is True, also closes the file handle. + """ + self.file_handle.write(']') + if self.own_file_handle: + self.file_handle.close() + self.file_handle = None + + class MARCWriter(Writer): """ A class for writing MARC21 records in transmission format. diff --git a/test/writer.py b/test/writer.py index 510d73a..576196c 100644 --- a/test/writer.py +++ b/test/writer.py @@ -4,6 +4,245 @@ import textwrap from six import BytesIO, StringIO, u, binary_type +try: + # the json module was included in the stdlib in python 2.6 + # http://docs.python.org/library/json.html + import json +except ImportError: + # simplejson 2.0.9 is available for python 2.4+ + # http://pypi.python.org/pypi/simplejson/2.0.9 + # simplejson 1.7.3 is available for python 2.3+ + # http://pypi.python.org/pypi/simplejson/1.7.3 + import simplejson as json + + +class JSONWriterTest(unittest.TestCase): + + def test_own_file_handle_true(self): + """ + If a JSONWriter is created with own_file_handle = True, then when the + JSONWriter is closed the file handle is also closed. + """ + file_handle = StringIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.JSONWriter(file_handle, own_file_handle = True) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertTrue( + file_handle.closed, + 'The file handle should close when the writer closes') + + def test_own_file_handle_false(self): + """ + If a JSONWriter is created with own_file_handle = False, then when the + JSONWriter is closed the file handle is NOT also closed. + """ + file_handle = StringIO() + self.assertFalse( + file_handle.closed, + 'The file handle should be open') + writer = pymarc.JSONWriter(file_handle, own_file_handle = False) + self.assertFalse( + file_handle.closed, + 'The file handle should still be open') + writer.close() + self.assertFalse( + file_handle.closed, + 'The file handle should NOT close when the writer closes') + + def test_writing_0_records(self): + expected = json.loads(r""" + [] + """) + file_handle = StringIO() + try: + writer = pymarc.JSONWriter(file_handle, own_file_handle = False) + writer.close() + actual = json.loads(file_handle.getvalue()) + self.assertEquals(actual, expected) + finally: + file_handle.close() + + def test_writing_empty_record(self): + expected = json.loads(r""" + [ + { + "leader" : " 22 4500", + "fields" : [] + } + ] + """) + file_handle = StringIO() + try: + writer = pymarc.JSONWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + writer.write(record) + writer.close() + actual = json.loads(file_handle.getvalue()) + self.assertEquals(actual, expected) + finally: + file_handle.close() + + def test_writing_1_record(self): + expected = json.loads(r""" + [ + { + "leader" : " 22 4500", + "fields" : [ + { + "100": { + "ind1": "0", + "ind2": "0", + "subfields": [ + { "a": "me" } + ] + } + }, + { + "245": { + "ind1": "0", + "ind2": "0", + "subfields": [ + { "a": "Foo /" }, + { "c": "by me." } + ] + } + } + ] + } + ] + """) + file_handle = StringIO() + try: + writer = pymarc.JSONWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + writer.close() + actual = json.loads(file_handle.getvalue()) + self.assertEquals(actual, expected) + finally: + file_handle.close() + + def test_writing_3_records(self): + expected = json.loads(r""" + [ + { + "leader" : " 22 4500", + "fields" : [ + { + "008": "090227s2009 mau chi d" + }, + { + "100": { + "ind1": "0", + "ind2": "0", + "subfields": [ + { "a": "me" } + ] + } + }, + { + "245": { + "ind1": "0", + "ind2": "0", + "subfields": [ + { "a": "Foo /" }, + { "c": "by me." } + ] + } + } + ] + }, + { + "leader" : " 22 4500", + "fields" : [ + { + "100": { + "ind1": "0", + "ind2": "0", + "subfields": [ + { "a": "me" } + ] + } + }, + { + "245": { + "ind1": "0", + "ind2": "0", + "subfields": [ + { "a": "Foo /" }, + { "c": "by me." } + ] + } + } + ] + }, + { + "leader" : " 22 4500", + "fields" : [ + { + "245": { + "ind1": "0", + "ind2": "0", + "subfields": [ + { "a": "Foo /" }, + { "c": "by me." } + ] + } + } + ] + } + ] + """) + file_handle = StringIO() + try: + writer = pymarc.JSONWriter(file_handle, own_file_handle = False) + record = pymarc.Record() + record.add_field( + pymarc.Field( + '008', + data=u('090227s2009 mau chi d'))) + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + record = pymarc.Record() + record.add_field( + pymarc.Field('100', ['0', '0'], ['a', u('me')])) + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + record = pymarc.Record() + record.add_field( + pymarc.Field( + '245', + ['0', '0'], + ['a', u('Foo /'), 'c', u('by me.')])) + writer.write(record) + writer.close() + actual = json.loads(file_handle.getvalue()) + self.assertEquals(actual, expected) + finally: + file_handle.close() + class MARCWriterTest(unittest.TestCase): @@ -396,10 +635,11 @@ def test_own_file_handle_false(self): def suite(): + json_suite = unittest.makeSuite(JSONWriterTest, 'test') marc_suite = unittest.makeSuite(MARCWriterTest, 'test') text_suite = unittest.makeSuite(TextWriterTest, 'test') xml_suite = unittest.makeSuite(XMLWriterTest, 'test') - test_suite = unittest.TestSuite((marc_suite, text_suite, xml_suite)) + test_suite = unittest.TestSuite((json_suite, marc_suite, text_suite, xml_suite)) return test_suite if __name__ == '__main__': From 653703605f928a294729b5903120fa9c90b64a5f Mon Sep 17 00:00:00 2001 From: Jim Nicholls Date: Sun, 6 Sep 2015 16:04:23 +1000 Subject: [PATCH 6/7] Refactor common functionality into Writer. --- pymarc/writer.py | 76 +++++++++++++++++------------------------------- 1 file changed, 26 insertions(+), 50 deletions(-) diff --git a/pymarc/writer.py b/pymarc/writer.py index bd6640d..f3c50b2 100644 --- a/pymarc/writer.py +++ b/pymarc/writer.py @@ -20,11 +20,23 @@ class Writer(object): + def __init__(self, file_handle, own_file_handle = True): + self.file_handle = file_handle + self.own_file_handle = own_file_handle + def write(self, record): - pass + if not isinstance(record, Record): + raise WriteNeedsRecord def close(self): - pass + """ + Closes the writer. + + If own_file_handle is True, also closes the file handle. + """ + if self.own_file_handle: + self.file_handle.close() + self.file_handle = None class JSONWriter(Writer): @@ -59,9 +71,7 @@ def __init__(self, file_handle, own_file_handle = True): closed when the writer is closed. Otherwise the file handle will be left open. """ - super(JSONWriter, self).__init__() - self.file_handle = file_handle - self.own_file_handle = own_file_handle + super(JSONWriter, self).__init__(file_handle, own_file_handle) self.write_count = 0 self.file_handle.write('[') @@ -69,8 +79,7 @@ def write(self, record): """ Writes a record. """ - if not isinstance(record, Record): - raise WriteNeedsRecord + Writer.write(self, record) if self.write_count > 0: self.file_handle.write(',') json.dump(record.as_dict(), self.file_handle, separators=(',', ':')) @@ -78,14 +87,12 @@ def write(self, record): def close(self): """ - Closes the file. + Closes the writer. If own_file_handle is True, also closes the file handle. """ self.file_handle.write(']') - if self.own_file_handle: - self.file_handle.close() - self.file_handle = None + Writer.close(self) class MARCWriter(Writer): @@ -115,28 +122,15 @@ def __init__(self, file_handle, own_file_handle = True): closed when the writer is closed. Otherwise the file handle will be left open. """ - super(MARCWriter, self).__init__() - self.file_handle = file_handle - self.own_file_handle = own_file_handle + super(MARCWriter, self).__init__(file_handle, own_file_handle) def write(self, record): """ Writes a record. """ - if not isinstance(record, Record): - raise WriteNeedsRecord + Writer.write(self, record) self.file_handle.write(record.as_marc()) - def close(self): - """ - Closes the file. - - If own_file_handle is True, also closes the file handle. - """ - if self.own_file_handle: - self.file_handle.close() - self.file_handle = None - class TextWriter(Writer): """ @@ -169,32 +163,19 @@ def __init__(self, file_handle, own_file_handle = True): closed when the writer is closed. Otherwise the file handle will be left open. """ - super(TextWriter, self).__init__() - self.file_handle = file_handle - self.own_file_handle = own_file_handle + super(TextWriter, self).__init__(file_handle, own_file_handle) self.write_count = 0 def write(self, record): """ Writes a record. """ - if not isinstance(record, Record): - raise WriteNeedsRecord + Writer.write(self, record) if self.write_count > 0: self.file_handle.write('\n') self.file_handle.write(str(record)) self.write_count += 1 - def close(self): - """ - Closes the file. - - If own_file_handle is True, also closes the file handle. - """ - if self.own_file_handle: - self.file_handle.close() - self.file_handle = None - class XMLWriter(Writer): """ @@ -229,9 +210,7 @@ def __init__(self, file_handle, own_file_handle = True): closed when the writer is closed. Otherwise the file handle will be left open. """ - super(XMLWriter, self).__init__() - self.file_handle = file_handle - self.own_file_handle = own_file_handle + super(XMLWriter, self).__init__(file_handle, own_file_handle) self.file_handle.write( b'') self.file_handle.write( @@ -241,18 +220,15 @@ def write(self, record): """ Writes a record. """ - if not isinstance(record, Record): - raise WriteNeedsRecord + Writer.write(self, record) node = pymarc.record_to_xml_node(record) self.file_handle.write(ET.tostring(node, encoding='utf-8')) def close(self): """ - Closes the file. + Closes the writer. If own_file_handle is True, also closes the file handle. """ self.file_handle.write(b'') - if self.own_file_handle: - self.file_handle.close() - self.file_handle = None + Writer.close(self) From 816b31ea75cbf2c3259ff4882fc42773a4bc3dfb Mon Sep 17 00:00:00 2001 From: Jim Nicholls Date: Sun, 6 Sep 2015 16:29:47 +1000 Subject: [PATCH 7/7] Revamp the documentation on the writers. --- pymarc/writer.py | 111 ++++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 49 deletions(-) diff --git a/pymarc/writer.py b/pymarc/writer.py index f3c50b2..7368859 100644 --- a/pymarc/writer.py +++ b/pymarc/writer.py @@ -46,21 +46,21 @@ class JSONWriter(Writer): IMPORTANT: You must the close a JSONWriter, otherwise you will not get valid JSON. - Simple usage: + Simple usage:: - from pymarc import JSONWriter + >>> from pymarc import JSONWriter - ## pass in a file - writer = JSONWriter(open('file.xml','wt')) - writer.write(record) - writer.close() + ### writing to a file + >>> writer = JSONWriter(open('file.json','wt')) + >>> writer.write(record) + >>> writer.close() # Important! - ## use StringIO if you want to write to a string - string = StringIO() - writer = JSONWriter(string, own_file_handle = False) - writer.write(record) - writer.close() - print string + ### writing to a string + >>> string = StringIO() + >>> writer = JSONWriter(string, own_file_handle = False) + >>> writer.write(record) + >>> writer.close() # Important! + >>> print string """ def __init__(self, file_handle, own_file_handle = True): @@ -99,24 +99,32 @@ class MARCWriter(Writer): """ A class for writing MARC21 records in transmission format. - Simple usage: + Simple usage:: - from pymarc import MARCWriter + >>> from pymarc import MARCWriter - ## pass in a file - writer = MARCWriter(file('file.dat','w')) - writer.write(record) + ### writing to a file + >>> writer = MARCWriter(open('file.dat','wb')) + >>> writer.write(record) + >>> writer.close() - ## use StringIO if you want to write to a string - string = StringIO() - writer = MARCWriter(string) - writer.write(record) - print string + ### writing to a string (Python 2 only) + >>> string = StringIO() + >>> writer = MARCWriter(string) + >>> writer.write(record) + >>> writer.close() + >>> print string + + ### writing to memory (Python 3 only) + >>> memory = BytesIO() + >>> writer = MARCWriter(memory) + >>> writer.write(record) + >>> writer.close() """ def __init__(self, file_handle, own_file_handle = True): """ - You need to pass in a file like object. + You need to pass in a byte file like object. If own_file_handle is True (the default) then the file handle will be closed when the writer is closed. Otherwise the file handle will be @@ -138,26 +146,26 @@ class TextWriter(Writer): A blank line separates each record. - Simple usage: + Simple usage:: - from pymarc import TextWriter + >>> from pymarc import TextWriter - ## pass in a file - writer = TextWriter(open('file.dat','wt')) - writer.write(record) - writer.close() + ### writing to a file + >>> writer = TextWriter(open('file.txt','wt')) + >>> writer.write(record) + >>> writer.close() - ## use StringIO if you want to write to a string - string = StringIO() - writer = MARCWriter(string, own_file_handle = False) - writer.write(record) - writer.close() - print string + ### writing to a string + >>> string = StringIO() + >>> writer = TextWriter(string, own_file_handle = False) + >>> writer.write(record) + >>> writer.close() + >>> print string """ def __init__(self, file_handle, own_file_handle = True): """ - You need to pass in a file like object. + You need to pass in a text file like object. If own_file_handle is True (the default) then the file handle will be closed when the writer is closed. Otherwise the file handle will be @@ -184,22 +192,27 @@ class XMLWriter(Writer): IMPORTANT: You must the close an XMLWriter, otherwise you will not get a valid XML document. - Simple usage: + Simple usage:: + + >>> from pymarc import XMLWriter - from pymarc import XMLWriter + ### writing to a file + >>> writer = XMLWriter(open('file.xml','wb')) + >>> writer.write(record) + >>> writer.close() # Important! - ## pass in a file - writer = XMLWriter(open('file.xml','wb')) - writer.write(record) - writer.close() + ### writing to a string (Python 2 only) + >>> string = StringIO() + >>> writer = XMLWriter(string) + >>> writer.write(record) + >>> writer.close() # Important! + >>> print string - ## use StringIO for Python 2 and BytesIO for Python 3 - ## if you want to write to a string/bytes. - string = StringIO() ## BytesIO() for Python 3 - writer = MARCWriter(string, own_file_handle = False) - writer.write(record) - writer.close() - print string + ### writing to memory (Python 3 only) + >>> memory = BytesIO() + >>> writer = XMLWriter(memory) + >>> writer.write(record) + >>> writer.close() # Important! """ def __init__(self, file_handle, own_file_handle = True):