From 9f2c137b0b2edd66938cc469d6b352ccd55e1e35 Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Thu, 13 Nov 2025 09:31:17 -0800 Subject: [PATCH 1/8] fix: any_value float -> str float_precision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Constructing a new Series with an unspecified datatype and string supertype led to a naive translation of passed floats to strings that was constrained by `polars.Config(float_precision=…)` Float64 & Float32 are now handled in the same fashion as `PySeries.cast` when constructing new Series. --- crates/polars-core/src/series/any_value.rs | 13 +++++++++++++ .../unit/constructors/test_any_value_fallbacks.py | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/crates/polars-core/src/series/any_value.rs b/crates/polars-core/src/series/any_value.rs index 5ec1890d6b9d..242794fdb663 100644 --- a/crates/polars-core/src/series/any_value.rs +++ b/crates/polars-core/src/series/any_value.rs @@ -1,6 +1,7 @@ use std::fmt::Write; use arrow::bitmap::MutableBitmap; +use polars_compute::cast::SerPrimitive; #[cfg(feature = "dtype-categorical")] use crate::chunked_array::builder::CategoricalChunkedBuilder; @@ -285,6 +286,18 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult builder.append_value(s), AnyValue::Null => builder.append_null(), AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(), + AnyValue::Float64(f) => { + let mut tmp = vec![]; + SerPrimitive::write(&mut tmp, *f); + let s = std::str::from_utf8(&tmp).unwrap(); + builder.append_value(s); + }, + AnyValue::Float32(f) => { + let mut tmp = vec![]; + SerPrimitive::write(&mut tmp, *f as f64); // promote to f64 for serialization + let s = std::str::from_utf8(&tmp).unwrap(); + builder.append_value(s); + }, av => { owned.clear(); write!(owned, "{av}").unwrap(); diff --git a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py index f189a572c766..4c896bf4fec4 100644 --- a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py +++ b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py @@ -408,3 +408,11 @@ def test_categorical_lit_18874() -> None: ] ), ) + + +def test_float_to_string_precision_25257() -> None: + with pl.Config(float_precision=1): + s = pl.Series(["", 0.123, 0.123456789], strict=False) + + # Float64 should have ~17 digits of precision preserved in its string repr + assert (s[1:] == pl.Series(["0.123", "0.123456789"])).all() From 4abd3aaa29736d216ee97808f4a40857340ab940 Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Thu, 13 Nov 2025 13:17:30 -0800 Subject: [PATCH 2/8] reuse float buffer in any_values_to_string_nonstrict --- crates/polars-core/src/series/any_value.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/polars-core/src/series/any_value.rs b/crates/polars-core/src/series/any_value.rs index 242794fdb663..4536ccfd18d7 100644 --- a/crates/polars-core/src/series/any_value.rs +++ b/crates/polars-core/src/series/any_value.rs @@ -280,6 +280,7 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult StringChunked { let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len()); let mut owned = String::new(); // Amortize allocations. + let mut float_buf = vec![]; for av in values { match av { AnyValue::String(s) => builder.append_value(s), @@ -287,15 +288,15 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult builder.append_null(), AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(), AnyValue::Float64(f) => { - let mut tmp = vec![]; - SerPrimitive::write(&mut tmp, *f); - let s = std::str::from_utf8(&tmp).unwrap(); + float_buf.clear(); + SerPrimitive::write(&mut float_buf, *f); + let s = std::str::from_utf8(&float_buf).unwrap(); builder.append_value(s); }, AnyValue::Float32(f) => { - let mut tmp = vec![]; - SerPrimitive::write(&mut tmp, *f as f64); // promote to f64 for serialization - let s = std::str::from_utf8(&tmp).unwrap(); + float_buf.clear(); + SerPrimitive::write(&mut float_buf, *f as f64); // promote to f64 for serialization + let s = std::str::from_utf8(&float_buf).unwrap(); builder.append_value(s); }, av => { From a0789035bcc95753789f01d20c0a603dffbcd511 Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Fri, 14 Nov 2025 08:24:41 -0800 Subject: [PATCH 3/8] add nested structures to test_float_to_string_precision testq --- .../constructors/test_any_value_fallbacks.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py index 4c896bf4fec4..8ef4517303e8 100644 --- a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py +++ b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py @@ -410,9 +410,17 @@ def test_categorical_lit_18874() -> None: ) -def test_float_to_string_precision_25257() -> None: +@pytest.mark.parametrize( + ("values", "expected"), + [ + # Float64 should have ~17; Float32 ~6 digits of precision preserved in its string repr + (["", 0.123, 0.123456789], ["", "0.123", "0.123456789"]), + ([{"a": ""}, {"a": 0.123}, {"a": 0.123456789}], [{"a": ""}, {"a": "0.123"}, {"a": "0.123456789"}]), + ([[""], [0.123], [0.123456789]], [[""], ["0.123"], ["0.123456789"]]), + ] +) +def test_float_to_string_precision_25257(values, expected) -> None: with pl.Config(float_precision=1): - s = pl.Series(["", 0.123, 0.123456789], strict=False) + s = pl.Series(values, strict=False) - # Float64 should have ~17 digits of precision preserved in its string repr - assert (s[1:] == pl.Series(["0.123", "0.123456789"])).all() + assert (s == pl.Series(expected)).all() From 274c574f66a5631dbe41efd5e300809ad9d233c6 Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Fri, 14 Nov 2025 08:32:06 -0800 Subject: [PATCH 4/8] add typing to test_float_to_string_precision_25257 --- .../unit/constructors/test_any_value_fallbacks.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py index 8ef4517303e8..533ea3a3ea39 100644 --- a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py +++ b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py @@ -413,13 +413,18 @@ def test_categorical_lit_18874() -> None: @pytest.mark.parametrize( ("values", "expected"), [ - # Float64 should have ~17; Float32 ~6 digits of precision preserved in its string repr + # Float64 should have ~17; Float32 ~6 digits of precision preserved (["", 0.123, 0.123456789], ["", "0.123", "0.123456789"]), - ([{"a": ""}, {"a": 0.123}, {"a": 0.123456789}], [{"a": ""}, {"a": "0.123"}, {"a": "0.123456789"}]), + ( + [{"a": ""}, {"a": 0.123}, {"a": 0.123456789}], + [{"a": ""}, {"a": "0.123"}, {"a": "0.123456789"}], + ), ([[""], [0.123], [0.123456789]], [[""], ["0.123"], ["0.123456789"]]), - ] + ], ) -def test_float_to_string_precision_25257(values, expected) -> None: +def test_float_to_string_precision_25257( + values: list[Any], expected: list[Any] +) -> None: with pl.Config(float_precision=1): s = pl.Series(values, strict=False) From 65ec1d898afbc9e4431111d7321647880ab5e71d Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Tue, 18 Nov 2025 08:51:59 -0800 Subject: [PATCH 5/8] fix any_values_to_string recurse nested types --- crates/polars-core/src/series/any_value.rs | 65 +++++++++++++++++-- .../constructors/test_any_value_fallbacks.py | 27 +++++++- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/crates/polars-core/src/series/any_value.rs b/crates/polars-core/src/series/any_value.rs index 4536ccfd18d7..fd1fba0deeed 100644 --- a/crates/polars-core/src/series/any_value.rs +++ b/crates/polars-core/src/series/any_value.rs @@ -278,30 +278,87 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult StringChunked { + fn _write_any_value(av: &AnyValue<'_>, buffer: &mut String, float_buf: &mut Vec) { + match av { + AnyValue::String(s) => buffer.push_str(s), + AnyValue::Float64(f) => { + float_buf.clear(); + SerPrimitive::write(float_buf, *f); + let s = std::str::from_utf8(&float_buf).unwrap(); + buffer.push_str(s); + }, + AnyValue::Float32(f) => { + float_buf.clear(); + SerPrimitive::write(float_buf, *f as f64); + let s = std::str::from_utf8(&float_buf).unwrap(); + buffer.push_str(s); + }, + AnyValue::StructOwned(payload) => { + buffer.push('{'); + let mut iter = payload.0.iter().peekable(); + while let Some(child) = iter.next() { + _write_any_value(child, buffer, float_buf); + if iter.peek().is_some() { + buffer.push(',') + } + } + buffer.push('}'); + }, + AnyValue::Struct(_, _, flds) => { + let mut vals = Vec::with_capacity(flds.len()); + av._materialize_struct_av(&mut vals); + + buffer.push('{'); + let mut iter = vals.iter().peekable(); + while let Some(child) = iter.next() { + _write_any_value(child, buffer, float_buf); + if iter.peek().is_some() { + buffer.push(',') + } + } + buffer.push('}'); + }, + AnyValue::List(vals) | AnyValue::Array(vals, _) => { + buffer.push('['); + let mut iter = vals.iter().peekable(); + while let Some(child) = iter.next() { + _write_any_value(&child, buffer, float_buf); + if iter.peek().is_some() { + buffer.push(','); + } + } + buffer.push(']'); + }, + av => { + write!(buffer, "{av}").unwrap(); + }, + } + } + let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, values.len()); let mut owned = String::new(); // Amortize allocations. let mut float_buf = vec![]; for av in values { + owned.clear(); + float_buf.clear(); + match av { AnyValue::String(s) => builder.append_value(s), AnyValue::StringOwned(s) => builder.append_value(s), AnyValue::Null => builder.append_null(), AnyValue::Binary(_) | AnyValue::BinaryOwned(_) => builder.append_null(), AnyValue::Float64(f) => { - float_buf.clear(); SerPrimitive::write(&mut float_buf, *f); let s = std::str::from_utf8(&float_buf).unwrap(); builder.append_value(s); }, AnyValue::Float32(f) => { - float_buf.clear(); SerPrimitive::write(&mut float_buf, *f as f64); // promote to f64 for serialization let s = std::str::from_utf8(&float_buf).unwrap(); builder.append_value(s); }, av => { - owned.clear(); - write!(owned, "{av}").unwrap(); + _write_any_value(av, &mut owned, &mut float_buf); builder.append_value(&owned); }, } diff --git a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py index 533ea3a3ea39..85c3a9d66259 100644 --- a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py +++ b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any import pytest +from numpy import array import polars as pl from polars._plr import PySeries @@ -415,16 +416,36 @@ def test_categorical_lit_18874() -> None: [ # Float64 should have ~17; Float32 ~6 digits of precision preserved (["", 0.123, 0.123456789], ["", "0.123", "0.123456789"]), + (["", [0.123, 0.123456789]], ["", "[0.123,0.123456789]"]), + (["", array([0.123, 0.123456789])], ["", "[0.123,0.123456789]"]), + (["", {"a": 0.123, "b": 0.123456789}], ["", "{0.123,0.123456789}"]), + (["", [{"a": 0.123, "b": 0.123456789}]], ["", "[{0.123,0.123456789}]"]), + (["", {"x": [0.1, 0.2]}, [{"y": 0.3}]], ["", "{[0.1,0.2]}", "[{0.3}]"]), ( - [{"a": ""}, {"a": 0.123}, {"a": 0.123456789}], - [{"a": ""}, {"a": "0.123"}, {"a": "0.123456789"}], + ["", None, {"a": None, "b": 1.0}, [None, 2.0]], + ["", None, "{null,1.0}", "[null,2.0]"], ), - ([[""], [0.123], [0.123456789]], [[""], ["0.123"], ["0.123456789"]]), + (["", [], {}], ["", "[]", "{}"]), + (["", [0.5]], ["", "[0.5]"]), + (["", {"a": 0.5}], ["", "{0.5}"]), + ], + ids=[ + "basic_floats", + "nested_list", + "nested_array", + "basic_struct", + "list_of_structs", + "nested_mixed", + "mixed_nulls", + "empty_containers", + "single_element_list", + "single_element_struct", ], ) def test_float_to_string_precision_25257( values: list[Any], expected: list[Any] ) -> None: + # verify the conversion is decoupled from Display formatting with pl.Config(float_precision=1): s = pl.Series(values, strict=False) From de6fa7db703046816cc9b6b78746fe872ff38c09 Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Tue, 18 Nov 2025 09:52:08 -0800 Subject: [PATCH 6/8] ref: test_float_to_string_precision simplify args --- .../constructors/test_any_value_fallbacks.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py index 85c3a9d66259..ddae7130da10 100644 --- a/py-polars/tests/unit/constructors/test_any_value_fallbacks.py +++ b/py-polars/tests/unit/constructors/test_any_value_fallbacks.py @@ -415,19 +415,19 @@ def test_categorical_lit_18874() -> None: ("values", "expected"), [ # Float64 should have ~17; Float32 ~6 digits of precision preserved - (["", 0.123, 0.123456789], ["", "0.123", "0.123456789"]), - (["", [0.123, 0.123456789]], ["", "[0.123,0.123456789]"]), - (["", array([0.123, 0.123456789])], ["", "[0.123,0.123456789]"]), - (["", {"a": 0.123, "b": 0.123456789}], ["", "{0.123,0.123456789}"]), - (["", [{"a": 0.123, "b": 0.123456789}]], ["", "[{0.123,0.123456789}]"]), - (["", {"x": [0.1, 0.2]}, [{"y": 0.3}]], ["", "{[0.1,0.2]}", "[{0.3}]"]), + ([0.123, 0.123456789], ["0.123", "0.123456789"]), + ([[0.123, 0.123456789]], ["[0.123,0.123456789]"]), + ([array([0.123, 0.123456789])], ["[0.123,0.123456789]"]), + ([{"a": 0.123, "b": 0.123456789}], ["{0.123,0.123456789}"]), + ([[{"a": 0.123, "b": 0.123456789}]], ["[{0.123,0.123456789}]"]), + ([{"x": [0.1, 0.2]}, [{"y": 0.3}]], ["{[0.1,0.2]}", "[{0.3}]"]), ( - ["", None, {"a": None, "b": 1.0}, [None, 2.0]], - ["", None, "{null,1.0}", "[null,2.0]"], + [None, {"a": None, "b": 1.0}, [None, 2.0]], + [None, "{null,1.0}", "[null,2.0]"], ), - (["", [], {}], ["", "[]", "{}"]), - (["", [0.5]], ["", "[0.5]"]), - (["", {"a": 0.5}], ["", "{0.5}"]), + ([[], {}], ["[]", "{}"]), + ([[0.5]], ["[0.5]"]), + ([{"a": 0.5}], ["{0.5}"]), ], ids=[ "basic_floats", @@ -447,6 +447,6 @@ def test_float_to_string_precision_25257( ) -> None: # verify the conversion is decoupled from Display formatting with pl.Config(float_precision=1): - s = pl.Series(values, strict=False) + s = pl.Series(values, strict=False, dtype=pl.String) assert (s == pl.Series(expected)).all() From 2dfa178b04e68f79b98b0eb24060ed74382e434f Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Tue, 18 Nov 2025 10:36:42 -0800 Subject: [PATCH 7/8] fix rust any_values_to_string clippy errors --- crates/polars-core/src/series/any_value.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/polars-core/src/series/any_value.rs b/crates/polars-core/src/series/any_value.rs index fd1fba0deeed..825ce14be335 100644 --- a/crates/polars-core/src/series/any_value.rs +++ b/crates/polars-core/src/series/any_value.rs @@ -284,13 +284,13 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult { float_buf.clear(); SerPrimitive::write(float_buf, *f); - let s = std::str::from_utf8(&float_buf).unwrap(); + let s = std::str::from_utf8(float_buf).unwrap(); buffer.push_str(s); }, AnyValue::Float32(f) => { float_buf.clear(); SerPrimitive::write(float_buf, *f as f64); - let s = std::str::from_utf8(&float_buf).unwrap(); + let s = std::str::from_utf8(float_buf).unwrap(); buffer.push_str(s); }, AnyValue::StructOwned(payload) => { From 1c38e3bb72bbe64325c51c27d8ca35b1ac05cb6f Mon Sep 17 00:00:00 2001 From: Cameron Riddell Date: Tue, 18 Nov 2025 11:30:30 -0800 Subject: [PATCH 8/8] fix any_values_to_string dtypes cfg --- crates/polars-core/src/series/any_value.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/polars-core/src/series/any_value.rs b/crates/polars-core/src/series/any_value.rs index 825ce14be335..1619306ec38c 100644 --- a/crates/polars-core/src/series/any_value.rs +++ b/crates/polars-core/src/series/any_value.rs @@ -293,6 +293,7 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult { buffer.push('{'); let mut iter = payload.0.iter().peekable(); @@ -304,6 +305,7 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult { let mut vals = Vec::with_capacity(flds.len()); av._materialize_struct_av(&mut vals); @@ -318,7 +320,19 @@ fn any_values_to_string(values: &[AnyValue], strict: bool) -> PolarsResult { + #[cfg(feature = "dtype-array")] + AnyValue::Array(vals, _) => { + buffer.push('['); + let mut iter = vals.iter().peekable(); + while let Some(child) = iter.next() { + _write_any_value(&child, buffer, float_buf); + if iter.peek().is_some() { + buffer.push(','); + } + } + buffer.push(']'); + }, + AnyValue::List(vals) => { buffer.push('['); let mut iter = vals.iter().peekable(); while let Some(child) = iter.next() {