From ffdbc3eb078e544c61eb99e860264af262522d31 Mon Sep 17 00:00:00 2001 From: zaryab-ali Date: Fri, 20 Sep 2024 22:06:01 +0500 Subject: [PATCH 1/2] added support for ns in primitie function and added the abilty to downcast --- pyiceberg/io/pyarrow.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 999813d0c2..c5411d14d1 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1068,8 +1068,19 @@ def primitive(self, primitive: pa.DataType) -> PrimitiveType: return StringType() elif pa.types.is_date32(primitive): return DateType() - elif isinstance(primitive, pa.Time64Type) and primitive.unit == "us": + elif isinstance(primitive, pa.Time64Type) and (primitive.unit == "us" or primitive.unit == "ns"): + primitive = cast(pa.TimestampType, primitive) + if primitive.unit == "ns": + if self._downcast_ns_timestamp_to_us: + logger.warning("Iceberg does not yet support 'ns' timestamp precision. Downcasting to 'us'.") + else: + raise TypeError( + "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us-on-write' configuration property to automatically downcast 'ns' to 'us' on write." + ) + else: + raise TypeError(f"Unsupported precision for timestamp type: {primitive.unit}") return TimeType() + elif pa.types.is_timestamp(primitive): primitive = cast(pa.TimestampType, primitive) if primitive.unit in ("s", "ms", "us"): From 5c3483b25d8655067930bb8ffe938a6296c6ead6 Mon Sep 17 00:00:00 2001 From: zaryab-ali Date: Fri, 20 Sep 2024 22:20:27 +0500 Subject: [PATCH 2/2] added support for ns in primitie function, also added the abilty to downcast --- pyiceberg/io/pyarrow.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index c5411d14d1..f5d468cd17 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1077,8 +1077,6 @@ def primitive(self, primitive: pa.DataType) -> PrimitiveType: raise TypeError( "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us-on-write' configuration property to automatically downcast 'ns' to 'us' on write." ) - else: - raise TypeError(f"Unsupported precision for timestamp type: {primitive.unit}") return TimeType() elif pa.types.is_timestamp(primitive):