Skip to content

Commit

Permalink
Closes #2054: Fix iconv encoding for multi-byte characters and reenab…
Browse files Browse the repository at this point in the history
…le tests (#2055)

* Fix iconv encoding for multi-byte characters
In the initial encoding implementation, the output buffer for
calculating the buffer length was a `bytes` buffer and that
was causing issues when an output character was a multi-byte
UTF-8 character. To fix this, the `bytes` buffer has been
switched to a `uint-8` array.

* Re-enable IDNA tests
  • Loading branch information
bmcdonald3 authored Jan 11, 2023
1 parent b484cd3 commit b023f48
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 3 deletions.
2 changes: 1 addition & 1 deletion src/Codecs.chpl
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ module Codecs {
var inSize = (inBufSize): c_size_t;

// TODO: this is probably worst way to allocate this
var chplRes:bytes = (" "*(inBufSize*4));
var chplRes: [0..#(inBufSize*4)] uint(8) = 0x00;
var origSize = chplRes.size;
var outSize = chplRes.size: c_size_t;

Expand Down
2 changes: 0 additions & 2 deletions tests/string_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,6 @@ def test_get_fixes(self):
p = strings.get_suffixes(1, return_origins=False, proper=False)
self.assertListEqual(["c", "d", "i"], p.to_list())

@pytest.mark.skip(reason="Skipping until issue resolved.")
def test_encoding(self):
idna_strings = ak.array(['Bücher.example','ドメイン.テスト', 'домен.испытание', 'Königsgäßchen'])
expected = ak.array(['xn--bcher-kva.example','xn--eckwd4c7c.xn--zckzah', 'xn--d1acufc.xn--80akhbyknj4f', 'xn--knigsgchen-b4a3dun'])
Expand All @@ -666,7 +665,6 @@ def test_encoding(self):
a3 = ak.random_strings_uniform(1, 10, UNIQUE, characters="printable")
self.assertTrue((a3 == a3.encode('ascii').decode('ascii')).all())

@pytest.mark.skip(reason="Skipping until issue resolved.")
def test_idna_utf16(self):
s = ak.array(['xn--mnchen-3ya', 'xn--zrich-kva', 'example.com'])

Expand Down

0 comments on commit b023f48

Please sign in to comment.