From 5369e5a9d9f960f3315b13581a149dc2fd8312d7 Mon Sep 17 00:00:00 2001 From: Alexis Jeandet Date: Sun, 3 Dec 2023 14:58:08 +0100 Subject: [PATCH 1/3] Fixes #26, NumElements must always be > 0 even with empty strings With empty strings, NumElements is 1 and we store a \0 char. Signed-off-by: Alexis Jeandet --- include/cdfpp/cdf-io/saving/create_records.hpp | 15 +++++++++------ pycdfpp/attribute.hpp | 11 +++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/include/cdfpp/cdf-io/saving/create_records.hpp b/include/cdfpp/cdf-io/saving/create_records.hpp index e8f06cfc..d1066738 100644 --- a/include/cdfpp/cdf-io/saving/create_records.hpp +++ b/include/cdfpp/cdf-io/saving/create_records.hpp @@ -60,6 +60,13 @@ namespace saving return cpr; } + int32_t attribute_entry_num_elements(const data_t& entry) + { + return visit( + entry, [](const cdf_none&) -> int32_t { return 0; }, + [](const auto& v) -> int32_t { return std::size(v); }); + } + inline void create_file_attributes_records(const CDF& cdf, saving_context& svg_ctx) { for (const auto& [name, attribute] : cdf.attributes) @@ -96,9 +103,7 @@ namespace saving }, [](const auto&) -> uint32_t { return 0; }); } - aedr.record.NumElements = visit( - data, [](const cdf_none&) -> uint32_t { return 0; }, - [](const auto& v) -> uint32_t { return std::size(v); }); + aedr.record.NumElements = attribute_entry_num_elements(data); value_index += 1; update_size(aedr, aedr.record.NumElements * cdf_type_size(aedr.record.DataType)); } @@ -143,9 +148,7 @@ namespace saving }, [](const auto&) -> int32_t { return 0; }); } - aedr.record.NumElements = visit( - data, [](const cdf_none&) -> int32_t { return 0; }, - [](const auto& v) -> int32_t { return std::size(v); }); + aedr.record.NumElements = attribute_entry_num_elements(data); update_size(aedr, aedr.record.NumElements * cdf_type_size(aedr.record.DataType)); vac.adr.record.MAXzEntries = std::max(vac.adr.record.MAXzEntries, aedr.record.Num); vac.adr.record.NzEntries = std::size(vac.aedrs); diff --git a/pycdfpp/attribute.hpp b/pycdfpp/attribute.hpp index dc53a018..d86147e2 100644 --- a/pycdfpp/attribute.hpp +++ b/pycdfpp/attribute.hpp @@ -201,7 +201,14 @@ data_t to_attr_data_entry(const std::string& values, CDF_Types data_type) { if (data_type == CDF_Types::CDF_CHAR or data_type == CDF_Types::CDF_UCHAR) { - return { no_init_vector(std::cbegin(values), std::cend(values)), data_type }; + if (std::size(values) != 0) + { + return { no_init_vector(std::cbegin(values), std::cend(values)), data_type }; + } + else + { + return { no_init_vector { { 0 } }, data_type }; + } } else { @@ -229,7 +236,7 @@ data_t _time_to_data_t(const py::buffer& buffer) template data_t time_to_data_t(const std::vector& values) { - return data_t {no_init_vector(std::cbegin(values), std::cend(values))}; + return data_t { no_init_vector(std::cbegin(values), std::cend(values)) }; } template From 093f22c42867d7d1356c88ac96e45397d79b0740 Mon Sep 17 00:00:00 2001 From: Alexis Jeandet Date: Sun, 3 Dec 2023 15:02:38 +0100 Subject: [PATCH 2/3] Fixes typos pointed in #24 Signed-off-by: Alexis Jeandet --- README.md | 2 +- include/cdfpp/variable.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ba28d7ee..2e73057f 100644 --- a/README.md +++ b/README.md @@ -222,4 +222,4 @@ int main(int argc, char** argv) ``` ## caveats -- NRZ variables shape, in order to expose a consistent shape, PyCDFpp exposes the reccord count as first dimension and thus its value will be either 0 or 1 (0 mean empty variable). +- NRV variables shape, in order to expose a consistent shape, PyCDFpp exposes the reccord count as first dimension and thus its value will be either 0 or 1 (0 mean empty variable). diff --git a/include/cdfpp/variable.hpp b/include/cdfpp/variable.hpp index 7d4818a8..de46fd86 100644 --- a/include/cdfpp/variable.hpp +++ b/include/cdfpp/variable.hpp @@ -219,7 +219,7 @@ struct Variable stream_collection(os, shape(), ", "); os << "\n" << indent + 2 << "type: " << cdf_type_str(type()) << "\n" - << indent + 2 << "record varry: " << (is_nrv() ? "Flase" : "True") << "\n" + << indent + 2 << "record vary: " << (is_nrv() ? "False" : "True") << "\n" << indent + 2 << compression_type() << "\n\n"; os << indent + 2 << "Attributes:\n"; std::for_each(std::cbegin(attributes), std::cend(attributes), @@ -230,7 +230,7 @@ struct Variable os << indent << name() << ": "; stream_collection(os, shape(), ", "); os << ", [" << cdf_type_str(type()) - << "], record varry:" << (is_nrv() ? "Flase" : "True") + << "], record vary:" << (is_nrv() ? "False" : "True") << ", compression: " << cdf_compression_type_str(compression_type()) << std::endl; } return os; From 1d1e10f603809bb635bf68ac6ef0e2342cf2912b Mon Sep 17 00:00:00 2001 From: Alexis Jeandet Date: Sun, 3 Dec 2023 15:54:47 +0100 Subject: [PATCH 3/3] Uses hephaistos IP to solve GH Actions DNS issue Signed-off-by: Alexis Jeandet --- tests/full_corpus/test_full_corpus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/full_corpus/test_full_corpus.py b/tests/full_corpus/test_full_corpus.py index 5edacf1b..ec3dbb8d 100644 --- a/tests/full_corpus/test_full_corpus.py +++ b/tests/full_corpus/test_full_corpus.py @@ -59,7 +59,7 @@ def tearDown(self): ) @ddt.unpack def test_opens_in_memory_remote_files(self, fname, variables, attrs): - cdf = pycdfpp.load(requests.get(f"https://hephaistos.lpp.polytechnique.fr/data/mirrors/CDF/test_files/{fname}").content) + cdf = pycdfpp.load(requests.get(f"https://129.104.27.7/data/mirrors/CDF/test_files/{fname}", verify=False).content) self.assertIsNotNone(cdf) self.assertEqual(len(cdf), variables) self.assertEqual(len(cdf.attributes), attrs)