Sanitizers fixes plus specific handling across 2.x versions

Actually the CDF++ lib handles 3 cases:
3.x files
2.5.x+ files
2.4.x- files

Signed-off-by: Alexis Jeandet <alexis.jeandet@member.fsf.org>

include/cdfpp/cdf-io/common.hpp

@@ -57,11 +57,6 @@ inline bool is_v3x(const magic_numbers_t& magic)
     return cdf_version(magic).first >= 3;
 }
 
-inline bool is_v2_5_or_more(const magic_numbers_t& magic)
-{
-    return cdf_version(magic).first == 2 && cdf_version(magic).second >= 5;
-}
-
 inline bool is_cdf(const magic_numbers_t& magic_numbers) noexcept
 {
     return (((magic_numbers.first & 0xfff00000) == 0xCDF00000)

include/cdfpp/cdf-io/desc-records.hpp

@@ -33,6 +33,10 @@ struct v3x_tag
 {
 };
 
+struct v2x_tag
+{
+};
+
 struct v2_4_or_less_tag
 {
 };

include/cdfpp/cdf-io/loading/loading.hpp

@@ -83,6 +83,7 @@ namespace
         repr.majority = parsing_context.majority;
         repr.distribution_version = parsing_context.distribution_version();
         repr.compression_type = parsing_context.compression_type;
+        repr.lazy = lazy_load;
         if (!attribute::load_all<typename parsing_context_t::version_tag, iso_8859_1_to_utf8>(
                 parsing_context, repr))
             return std::nullopt;
@@ -113,12 +114,32 @@ namespace
             }
             return std::nullopt;
         }
-        else
+        else // Compression was introduced in CDF V2.6
         {
             auto parsing_ctx = make_parsing_context(
                 cdf_version_tag_t {}, std::move(buffer), cdf_compression_type::no_compression);
-            return impl_parse_cdf<common::with_iso_8859_1_to_utf8<iso_8859_1_to_utf8>>(
-                parsing_ctx, lazy_load);
+            if constexpr (!is_v3_v<cdf_version_tag_t>)
+            {
+                if (parsing_ctx.cdr.Release >= 5)
+                {
+                    auto new_ctx = make_parsing_context(v2_5_or_more_tag {},
+                        std::move(parsing_ctx.buffer), cdf_compression_type::no_compression);
+                    return impl_parse_cdf<common::with_iso_8859_1_to_utf8<iso_8859_1_to_utf8>>(
+                        new_ctx, lazy_load);
+                }
+                else
+                {
+                    auto new_ctx = make_parsing_context(v2_4_or_less_tag {},
+                        std::move(parsing_ctx.buffer), cdf_compression_type::no_compression);
+                    return impl_parse_cdf<common::with_iso_8859_1_to_utf8<iso_8859_1_to_utf8>>(
+                        new_ctx, lazy_load);
+                }
+            }
+            else
+            {
+                return impl_parse_cdf<common::with_iso_8859_1_to_utf8<iso_8859_1_to_utf8>>(
+                    parsing_ctx, lazy_load);
+            }
         }
     }
 
@@ -135,14 +156,9 @@ namespace
                 return parse_cdf<v3x_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
                     common::is_compressed(magic), lazy_load);
             }
-            else if (common::is_v2_5_or_more(magic))
-            {
-                return parse_cdf<v2_5_or_more_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
-                    common::is_compressed(magic), lazy_load);
-            }
             else
             {
-                return parse_cdf<v2_4_or_less_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
+                return parse_cdf<v2x_tag>(std::move(buffer), iso_8859_1_to_utf8_tag,
                     common::is_compressed(magic), lazy_load);
             }
         }

pycdfpp/variable.hpp

@@ -92,9 +92,16 @@ std::pair<data_t, typename Variable::shape_t> _numeric_to_nd_data_t(const py::bu
         throw std::invalid_argument { "Incompatible python and cdf types" };
     typename Variable::shape_t shape(info.ndim);
     std::copy(std::cbegin(info.shape), std::cend(info.shape), std::begin(shape));
-    no_init_vector<T> values(info.size);
-    std::memcpy(values.data(), info.ptr, info.size * sizeof(T));
-    return { data_t { std::move(values), data_type }, std::move(shape) };
+    if (info.size != 0)
+    {
+        no_init_vector<T> values(info.size);
+        std::memcpy(values.data(), info.ptr, info.size * sizeof(T));
+        return { data_t { std::move(values), data_type }, std::move(shape) };
+    }
+    else
+    {
+        return { data_t { no_init_vector<T>(0), data_type }, std::move(shape) };
+    }
 }
 
 template <CDF_Types data_type>
@@ -242,8 +249,8 @@ void def_variable_wrapper(T& mod)
         .def_property_readonly("values_encoded", make_values_view<true>, py::keep_alive<0, 1>())
         .def("_set_values", set_values, py::arg("values").noconvert(), py::arg("data_type"))
         .def("_add_attribute",
-            static_cast<VariableAttribute& (*)(Variable&, const std::string&, const string_or_buffer_t&,
-                CDF_Types)>(add_attribute),
+            static_cast<VariableAttribute& (*)(Variable&, const std::string&,
+                const string_or_buffer_t&, CDF_Types)>(add_attribute),
             py::arg { "name" }, py::arg { "values" }, py::arg { "data_type" },
             py::return_value_policy::reference_internal);
 }

tests/full_corpus/test_full_corpus.py

@@ -38,7 +38,7 @@ def tearDown(self):
     ( "cl_sp_edi_00000000_v01.cdf", 7, 39 ),
     ( "cluster-2_cp3drl_2002052000000_v1.cdf", 40, 32 ),
     ( "de_uv_sai_19910218_v01.cdf", 39, 12 ),
-    ( "ge_k0_cpi_19921231_v02.cdf", 1, 18 ),
+    ( "ge_k0_cpi_19921231_v02.cdf", 25, 18 ),
     ( "i1_av_ott_1983351130734_v01.cdf", 47, 18 ),
     ( "im_k0_euv_20011231_v01.cdf", 12, 13 ),
     ( "im_k0_rpi_20051218_v01.cdf", 25, 14 ),
@@ -59,6 +59,7 @@ def tearDown(self):
     )
     @ddt.unpack
     def test_opens_in_memory_remote_files(self, fname, variables, attrs):
+        print(fname)
         cdf = pycdfpp.load(requests.get(f"https://129.104.27.7/data/mirrors/CDF/test_files/{fname}", verify=False).content)
         self.assertIsNotNone(cdf)
         self.assertEqual(len(cdf), variables)

tests/simple_open/main.cpp

@@ -544,5 +544,29 @@ SCENARIO("Loading cdf files", "[CDF]")
                 REQUIRE(std::size(cd.variables) == 10);
             }
         }
+        WHEN("file is a 2.4.x cdf")
+        {
+            auto path = std::string(DATA_PATH) + "/ge_k0_cpi_19921231_v02.cdf";
+            REQUIRE(file_exists(path));
+            auto cd_opt = cdf::io::load(path);
+            REQUIRE(cd_opt != std::nullopt);
+            auto cd = *cd_opt;
+            THEN("All expected variables are found")
+            {
+                REQUIRE(std::size(cd.variables) == 25);
+            }
+        }
+        WHEN("file is a 2.5.x cdf")
+        {
+            auto path = std::string(DATA_PATH) + "/ac_h2_sis_20101105_v06.cdf";
+            REQUIRE(file_exists(path));
+            auto cd_opt = cdf::io::load(path);
+            REQUIRE(cd_opt != std::nullopt);
+            auto cd = *cd_opt;
+            THEN("All expected variables are found")
+            {
+                REQUIRE(std::size(cd.variables) == 61);
+            }
+        }
     }
 }