support CLR structs, both nullable and non-nullable #495 (#568)

* support CLR structs, both nullable and non-nullable #495

* release notes

.github/workflows/build.yml

@@ -2,7 +2,7 @@
 
 env:
   VERSION: 5.0.2
-  PACKAGE_SUFFIX: '-pre.1'
+  PACKAGE_SUFFIX: '-pre.2'
   # PACKAGE_SUFFIX: ''
   ASM_VERSION: 5.0.0
   DOC_INSTANCE: wrs/pq

docs/rn/5.0.2.md

@@ -4,6 +4,7 @@
 
 ### Improvements
 
+- Serialisation of CLR (not Parquet) structs and nullable structs is now properly handled and supported, thanks to @paulengineer.
 - For Windows, run unit tests on x86 and x32 explicitly.
 - Improved GHA build/release process, combining all workflows into one and simplifying it, most importantly release management.
 

src/Parquet.Test/Serialisation/ParquetSerializerTest.cs

@@ -1050,5 +1050,33 @@ public async Task DateOnlyTimeOnly_Nullable_Serde() {
         }
 
 #endif
+
+        private struct StructWithIntProp {
+            public int Id { get; set; }
+        }
+
+        private class ClassWithNullableCustomStruct {
+            public StructWithIntProp? NullableStruct { get; set; }
+        }
+
+        [Fact]
+        public async Task Class_With_Nullable_Struct() {
+
+            ParquetSchema schema = typeof(ClassWithNullableCustomStruct).GetParquetSchema(true);
+
+            var data = new List<ClassWithNullableCustomStruct> {
+                new ClassWithNullableCustomStruct() {
+                    NullableStruct = null
+                }
+            };
+
+            using var ms = new MemoryStream();
+            await ParquetSerializer.SerializeAsync(data, ms);
+
+            ms.Position = 0;
+            IList<ClassWithNullableCustomStruct> data2 = await ParquetSerializer.DeserializeAsync<ClassWithNullableCustomStruct>(ms);
+
+            Assert.Equivalent(data2, data);
+        }
     }
 }

src/Parquet.Test/Serialisation/SchemaReflectorTest.cs

@@ -662,5 +662,55 @@ public void Enums() {
             Assert.Equal(typeof(short), sedf.ClrType);
             Assert.False(dedf.IsNullable);
         }
+
+        struct SimpleClrStruct {
+            public int Id { get; set; }
+        }
+
+        [Fact]
+        public void ClrStruct_IsSupported() {
+            ParquetSchema schema = typeof(SimpleClrStruct).GetParquetSchema(true);
+            Assert.Single(schema.Fields);
+            Assert.Equal(typeof(int), schema.DataFields[0].ClrType);
+        }
+
+        class StructWithClrStruct {
+            public SimpleClrStruct S { get; set; }
+        }
+
+        [Fact]
+        public void ClrStruct_AsMember_IsSupported() {
+            ParquetSchema schema = typeof(StructWithClrStruct).GetParquetSchema(false);
+            Assert.Single(schema.Fields);
+
+            // check it's a required struct
+            StructField sf = (StructField)schema[0];
+            Assert.False(sf.IsNullable, "struct cannot be optional");
+
+            // check the struct field
+            Assert.Single(sf.Children);
+            var idField = (DataField)sf.Children[0];
+            Assert.Equal(typeof(int), idField.ClrType);
+        }
+
+        class StructWithNullableClrStruct {
+            // as CLR struct is ValueType, this resolves to System.Nullable<SimpleClrStruct>
+            public SimpleClrStruct? N { get; set; }
+        }
+
+        [Fact]
+        public void ClrStruct_AsNullableMember_IsSupported() {
+            ParquetSchema schema = typeof(StructWithNullableClrStruct).GetParquetSchema(false);
+            Assert.Single(schema.Fields);
+
+            // check it's a required struct
+            StructField sf = (StructField)schema[0];
+            Assert.True(sf.IsNullable, "struct must be nullable");
+
+            // check the struct field
+            Assert.Single(sf.Children);
+            var idField = (DataField)sf.Children[0];
+            Assert.Equal(typeof(int), idField.ClrType);
+        }
     }
 }

src/Parquet.sln

@@ -38,6 +38,7 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "rn", "rn", "{EE4ECD1F-5A44-4FC7-BC73-930AFB15D372}"
 	ProjectSection(SolutionItems) = preProject
 		..\docs\rn\5.0.2.md = ..\docs\rn\5.0.2.md
+		..\docs\rn\previous.md = ..\docs\rn\previous.md
 	EndProjectSection
 EndProject
 Global

src/Parquet/Extensions/TypeExtensions.cs

@@ -108,7 +108,7 @@ public static bool IsNullable(this Type t) {
             TypeInfo ti = t.GetTypeInfo();
 
             return
-                ti.IsClass ||
+                ti.IsClass || ti.IsInterface ||
                 (ti.IsGenericType && ti.GetGenericTypeDefinition() == typeof(Nullable<>));
         }
 
@@ -121,7 +121,7 @@ public static bool IsSystemNullable(this Type t) {
         public static Type GetNonNullable(this Type t) {
             TypeInfo ti = t.GetTypeInfo();
 
-            if(ti.IsClass) {
+            if(ti.IsClass || ti.IsInterface) {
                 return t;
             }
 

src/Parquet/Serialization/Dremel/FieldAssemblerCompiler.cs

@@ -304,20 +304,25 @@ private ClassMember GetClassMember(Type rootType, Expression rootVar,
 
                 // Dictionary is a special case, because it cannot be constructed independently in one go, so the client needs to know it a dictionary
 
-                Type? type = null;
-                MemberExpression? accessor = null;
+                Type type = rootType;
+                Expression accessor = rootVar;
 
-                PropertyInfo? pi = rootType.GetProperty(name);
+                if(type.IsSystemNullable()) {
+                    type = type.GetNonNullable();
+                    accessor = Expression.Property(accessor, "Value");
+                }
+
+                PropertyInfo? pi = type.GetProperty(name);
                 if(pi != null) {
                     type = pi.PropertyType;
-                    accessor = Expression.Property(rootVar, name);
+                    accessor = Expression.Property(accessor, name);
                 }
 
                 if(pi == null) {
                     FieldInfo? fi = rootType.GetField(name);
                     if(fi != null) {
                         type = fi.FieldType;
-                        accessor = Expression.Field(rootVar, name);
+                        accessor = Expression.Field(accessor, name);
                     }
                 }
 

src/Parquet/Serialization/Dremel/FieldStriperCompiler.cs

@@ -168,7 +168,9 @@ private Expression WhileBody(Expression element, bool isAtomic, int dl, Paramete
 
                 isAtomic
                     ? Expression.Assign(isLeafVar, Expression.Constant(true))
-                    : Expression.Assign(isLeafVar, elementType.IsValueType ? Expression.Constant(false) : valueVar.IsNull()),
+                    : (elementType.IsValueType && !elementType.IsSystemNullable())
+                        ? Expression.Assign(isLeafVar, Expression.Constant(false))
+                        : Expression.Assign(isLeafVar, valueVar.IsNull()),
 
                 Expression.IfThenElse(
                     Expression.IsTrue(isLeafVar),
@@ -248,19 +250,28 @@ private Expression GetClassMemberAccessorAndType(
                         Expression.Default(type)));
             }
 
-            PropertyInfo? pi = rootType.GetProperty(name);
-            if(pi != null) {
-                type = pi.PropertyType;
-                return Expression.Property(rootVar, name);
+            Expression? result = rootVar;
+            type = rootType;
+
+            if(rootType.IsSystemNullable()) {
+                result = Expression.Property(result, "Value");
+                type = rootType.GetNonNullable();
             }
 
-            FieldInfo? fi = rootType.GetField(name);
-            if(fi != null) {
+            PropertyInfo? pi = type.GetProperty(name);
+            FieldInfo? fi = type.GetField(name);
+
+            if(pi != null) {
+                type = pi.PropertyType;
+                result = Expression.Property(result, name);
+            } else if(fi != null) {
                 type = fi.FieldType;
-                return Expression.Field(rootVar, name);
+                result = Expression.Field(result, name);
+            } else {
+                throw new NotSupportedException($"There is no class property of field called '{name}'.");
             }
 
-            throw new NotSupportedException($"There is no class property of field called '{name}'.");
+            return result;
         }
 
         private Expression DissectRecord(

src/Parquet/Serialization/TypeExtensions.cs

@@ -242,20 +242,20 @@ private static Field MakeField(Type t, string columnName, string propertyName,
             ClassMember? member,
             bool forWriting) {
 
-            Type bt = t.IsNullable() ? t.GetNonNullable() : t;
-            if(member != null && member.IsLegacyRepeatable && !bt.IsGenericIDictionary() && bt.TryExtractIEnumerableType(out Type? bti)) {
-                bt = bti!;
+            Type baseType = t.IsNullable() ? t.GetNonNullable() : t;
+            if(member != null && member.IsLegacyRepeatable && !baseType.IsGenericIDictionary() && baseType.TryExtractIEnumerableType(out Type? bti)) {
+                baseType = bti!;
             }
 
-            if(SchemaEncoder.IsSupported(bt)) {
+            if(SchemaEncoder.IsSupported(baseType)) {
                 return ConstructDataField(columnName, propertyName, t, member);
             } else if(t.TryExtractDictionaryType(out Type? tKey, out Type? tValue)) {
                 return ConstructMapField(columnName, propertyName, tKey!, tValue!, forWriting);
             } else if(t.TryExtractIEnumerableType(out Type? elementType)) {
                 return ConstructListField(columnName, propertyName, elementType!, member, forWriting);
-            } else if(t.IsClass || t.IsInterface || t.IsValueType) {
-                // must be a struct then (c# class or c# struct)!
-                List<ClassMember> props = FindMembers(t, forWriting);
+            } else if(baseType.IsClass || baseType.IsInterface || baseType.IsValueType) {
+                // must be a struct then (c# class, interface or struct)
+                List<ClassMember> props = FindMembers(baseType, forWriting);
                 Field[] fields = props
                     .Select(p => MakeField(p, forWriting))
                     .Where(f => f != null)
@@ -268,6 +268,7 @@ private static Field MakeField(Type t, string columnName, string propertyName,
 
                 StructField sf = new StructField(columnName, fields);
                 sf.ClrPropName = propertyName;
+                sf.IsNullable = baseType.IsNullable() || t.IsSystemNullable();
                 return sf;
             }