Arrow output format for read operations

Detailed release notes to be written in PR description

Author: IvoDD

docs/mkdocs/docs/notebooks/ArcticDB_demo_read_as_arrow.ipynb

@@ -81,17 +81,14 @@ def __init__(
             Can be overridden by specifying the encoding version in the LibraryOptions argument to create_library.
 
         output_format: Union[OutputFormat, str], default = OutputFormat.PANDAS
-            Controls the default output format of all operations returning a dataframe.
-            The default behavior (OutputFormat.PANDAS) is to return `pandas.DataFrame`s or `pandas.Series` backed by
-            numpy arrays.
-            OutputFormat.EXPERIMENTAL_ARROW will return all dataframes as `pyarrow.Table`s. The arrow API is still
-            experimental and the arrow layout might change in a minor release.
-            Accepts the OutputFormat as either OutputFormat enum values or as case-insensitive strings like "pandas"
-            and "experimental_arrow".
-
-        arrow_string_format_default: Union[ArrowOutputStringFormat, "pa.DataType"] = ArrowOutputStringFormat.LARGE_STRING
-            Controls the default string format used for `OutputFormat.EXPERIMENTAL_ARROW`.
-            See documentation of `ArrowOutputStringFormat` for more information on the different options.
+            Default output format for all read operations on libraries created from this `Arctic` instance.
+            Can be overridden per library or per read operation.
+            See `OutputFormat` documentation for details on available formats.
+
+        arrow_string_format_default: Union[ArrowOutputStringFormat, "pa.DataType"], default = ArrowOutputStringFormat.LARGE_STRING
+            Default string column format when using `PYARROW` or `POLARS` output formats.
+            Can be overridden per library or per read operation.
+            See `ArrowOutputStringFormat` documentation for details on available string formats.
 
         Examples
         --------
@@ -196,14 +193,16 @@ def get_library(
             Unused if create_if_missing is False.
 
         output_format: Optional[Union[OutputFormat, str]], default = None
-            Controls the default output format of all operations on the library returning a dataframe.
-            For more information see documentation of `Arctic.__init__`.
-            If `None` uses the output format from the Arctic instance.
+            Default output format for all read operations on this library.
+            If `None`, uses the output format from the `Arctic` instance.
+            Can be overridden per read operation.
+            See `OutputFormat` documentation for details on available formats.
 
         arrow_string_format_default: Optional[Union[ArrowOutputStringFormat, "pa.DataType"]], default=None
-            If using `output_format=EXPERIMENTAL_ARROW` it sets the output format of string columns for arrow.
-            See documentation of `ArrowOutputStringFormat` for more information on the different options.
-            If `None` uses the default arrow_string_format from the `Library` instance.
+            Default string column format when using `PYARROW` or `POLARS` output formats on this library.
+            If `None`, uses the `arrow_string_format_default` from the `Arctic` instance.
+            Can be overridden per read operation.
+            See `ArrowOutputStringFormat` documentation for details on available string formats.
 
         Examples
         --------
@@ -268,14 +267,16 @@ def create_library(
             EnterpriseLibraryOptions. These options are only relevant to ArcticDB enterprise users.
 
         output_format: Optional[Union[OutputFormat, str]], default = None
-            Controls the default output format of all operations on the library returning a dataframe.
-            For more information see documentation of `Arctic.__init__`.
-            If `None` uses the output format from the Arctic instance.
+            Default output format for all read operations on this library.
+            If `None`, uses the output format from the `Arctic` instance.
+            Can be overridden per read operation.
+            See `OutputFormat` documentation for details on available formats.
 
         arrow_string_format_default: Optional[Union[ArrowOutputStringFormat, "pa.DataType"]], default=None
-            If using `output_format=EXPERIMENTAL_ARROW` it sets the output format of string columns for arrow.
-            See documentation of `ArrowOutputStringFormat` for more information on the different options.
-            If `None` uses the default arrow_string_format from the `Library` instance.
+            Default string column format when using `PYARROW` or `POLARS` output formats on this library.
+            If `None`, uses the `arrow_string_format_default` from the `Arctic` instance.
+            Can be overridden per read operation.
+            See `ArrowOutputStringFormat` documentation for details on available string formats.
 
         Examples
         --------

python/arcticdb/arctic.py

@@ -169,21 +169,44 @@ def __repr__(self):
 
 # TODO: Use enum.StrEnum when we no longer need to support python 3.9
 class OutputFormat(str, Enum):
+    """
+    Controls the output format of operations which return dataframes. All APIs which take an `output_format` argument
+    accept the enum values and case-insensitive strings. E.g. all of `OutputFormat.PYARROW`, `"PYARROW"`, `"pyarrow"`
+    will be interpreted as `OutputFormat.PYARROW`.
+
+    PANDAS (default):
+        Dataframes are returned as `pandas.DataFrame` or `pandas.Series` objects backed by numpy arrays.
+
+    PYARROW:
+        Dataframes are returned as `pyarrow.Table` objects using Apache Arrow's columnar memory format.
+        Both `PYARROW` and `POLARS` output formats use the same underlying Arrow memory layout, ensuring
+        zero-copy interoperability between them.
+        Provides better performance than `PANDAS`, especially for dataframes containing many string columns.
+        String format can be customized via `ArrowOutputStringFormat`.
+
+    POLARS:
+        Dataframes are returned as `polars.DataFrame` objects using Apache Arrow's columnar memory format.
+        Both `PYARROW` and `POLARS` output formats use the same underlying Arrow memory layout, ensuring
+        zero-copy interoperability between them.
+        Provides better performance than `PANDAS`, especially for dataframes containing many string columns.
+        String format can be customized via `ArrowOutputStringFormat`.
+    """
+
     PANDAS = "PANDAS"
-    EXPERIMENTAL_ARROW = "EXPERIMENTAL_ARROW"
-    EXPERIMENTAL_POLARS = "EXPERIMENTAL_POLARS"
+    PYARROW = "PYARROW"
+    POLARS = "POLARS"
 
 
 def output_format_to_internal(output_format: Union[OutputFormat, str]) -> InternalOutputFormat:
     if output_format.lower() == OutputFormat.PANDAS.lower():
         return InternalOutputFormat.PANDAS
-    elif output_format.lower() == OutputFormat.EXPERIMENTAL_ARROW.lower():
+    elif output_format.lower() == OutputFormat.PYARROW.lower():
         if not _PYARROW_AVAILABLE:
             raise ModuleNotFoundError(
                 "ArcticDB's pyarrow optional dependency missing but is required to use arrow output format."
             )
         return InternalOutputFormat.ARROW
-    elif output_format.lower() == OutputFormat.EXPERIMENTAL_POLARS.lower():
+    elif output_format.lower() == OutputFormat.POLARS.lower():
         if not _PYARROW_AVAILABLE or not _POLARS_AVAILABLE:
             raise ModuleNotFoundError(
                 "ArcticDB's pyarrow or polars optional dependencies are missing but are required to use polars output format."
@@ -195,24 +218,30 @@ def output_format_to_internal(output_format: Union[OutputFormat, str]) -> Intern
 
 class ArrowOutputStringFormat(str, Enum):
     """
-    Used to specify string format when output_format=OutputFormat.EXPERIMENTAL_ARROW.
-    Arguments allow specifying either the enum value or the corresponding pyarrow.DataType
+    Controls the string column format when using `PYARROW` or `POLARS` output formats.
+    Accepts either the enum value or the corresponding `pyarrow.DataType`.
 
     LARGE_STRING (default):
-    Produces string columns with type `pa.large_string()`. Total length of strings must fit in a 64-bit integer.
-    Does not deduplicate strings, so has better performance for columns with many unique strings.
+        Uses 64-bit variable-size encoding.
+        PyArrow: `pa.large_string()`, Polars: `pl.String`
+        Supports up to 2⁶³-1 bytes total string length per Arrow array.
+        Best for general-purpose use and when working with large string data.
 
     SMALL_STRING:
-    Produces string columns with type `pa.string()`. Total length of strings must fit in a 32-bit integer.
-    Does not deduplicate strings, so has better performance for columns with many unique strings.
-    Slightly faster than `LARGE_STRING` but does not work with very long strings.
+        Uses 32-bit variable-size encoding.
+        PyArrow: `pa.string()`, Polars: `pl.String`
+        Supports up to 2³¹-1 bytes total string length per Arrow array.
+        Slightly more memory efficient than `LARGE_STRING` when string data is known to be small.
 
     CATEGORICAL and DICTIONARY_ENCODED:
-    Both are different aliases for the same string format. Produces string columns with type
-    `pa.dictionary(pa.int32(), pa.large_string())`. Total length of strings must fit in a 64-bit integer.  Splitting in
-    record batches guarantees that 32-bit dictionary keys are sufficient.
-    Does deduplicate strings, so has better performance for columns with few unique strings.
-
+        Both are aliases for dictionary-encoded strings with int32 indices.
+        PyArrow: `pa.dictionary(pa.int32(), pa.large_string())`, Polars: `pl.Categorical`
+        Best for columns with low cardinality (few unique values repeated many times).
+        Deduplicates strings, reducing memory usage and improving performance when the number of
+        unique values is much smaller than the total number of rows.
+
+    For more details on physical layouts, see the Apache Arrow specification:
+    https://arrow.apache.org/docs/format/Columnar.html
     """
 
     CATEGORICAL = "CATEGORICAL"

python/arcticdb/options.py

@@ -25,7 +25,7 @@ def stringify_dictionary_encoded_columns(table, string_type=None):
 
 def convert_arrow_to_pandas_for_tests(table):
     """
-    Converts `pa.Table` outputted via `output_format=OutputFormat.EXPERIMENTAL_ARROW` to a `pd.DataFrame` so it would
+    Converts `pa.Table` outputted via `output_format=OutputFormat.PYARROW` to a `pd.DataFrame` so it would
     be identical to the one outputted via `output_format=OutputFormat.PANDAS`. This requires the following changes:
     - Replaces dictionary encoded string columns with regular string columns.
     - Fills null values in int columns with zeros.

python/arcticdb/util/arrow.py

@@ -792,7 +792,6 @@ def generate_original_column_names():
             pandas_meta = norm_meta.df.common
         elif input_type == "series":
             # For pandas series we always return a dataframe (to not lose the index information).
-            # TODO: Return a `pyarrow.Array` if index is not physically stored (Monday ref: 9360502457)
             pandas_meta = norm_meta.series.common
         elif input_type == "experimental_arrow":
             if norm_meta.experimental_arrow.has_index:

python/arcticdb/version_store/_normalization.py

@@ -385,7 +385,7 @@ def _set_allow_arrow_input(self, allow_arrow_input: bool = True):
 
     def _set_output_format_for_pipeline_tests(self, output_format):
         self.set_output_format(output_format)
-        if output_format == OutputFormat.EXPERIMENTAL_ARROW:
+        if output_format == OutputFormat.PYARROW:
             self._test_convert_arrow_back_to_pandas = True
 
     @classmethod
@@ -2728,8 +2728,8 @@ def _adapt_frame_data(self, frame_data, norm, output_format):
                 )
             if self._test_convert_arrow_back_to_pandas:
                 data = convert_arrow_to_pandas_for_tests(data)
-            if output_format.lower() == OutputFormat.EXPERIMENTAL_POLARS.lower():
-                data = pl.from_arrow(data)
+            if output_format.lower() == OutputFormat.POLARS.lower():
+                data = pl.from_arrow(data, rechunk=False)
         else:
             data = self._normalizer.denormalize(frame_data, norm)
             if norm.HasField("custom"):

python/arcticdb/version_store/_store.py

@@ -2026,17 +2026,17 @@ def read(
             on `LazyDataFrame` for more details.
 
         output_format: Optional[Union[OutputFormat, str]], default=None
-            Controls the output format of the result dataframe.
-            For more information see documentation of `Arctic.__init__`.
-            If `None` uses the default output format from the `Library` instance.
+            Output format for the returned dataframe.
+            If `None`, uses the output format from the `Library` instance.
+            See `OutputFormat` documentation for details on available formats.
 
         arrow_string_format_default: Optional[Union[ArrowOutputStringFormat, "pa.DataType"]], default=None
-            If using `output_format=EXPERIMENTAL_ARROW` it sets the output format of string columns for arrow.
-            See documentation of `ArrowOutputStringFormat` for more information on the different options.
-            If `None` uses the default arrow_string_format from the `Library` instance.
+            String column format when using `PYARROW` or `POLARS` output formats.
+            If `None`, uses the `arrow_string_format_default` from the `Library` instance.
+            See `ArrowOutputStringFormat` documentation for details on available string formats.
 
         arrow_string_format_per_column: Optional[Dict[str, Union[ArrowOutputStringFormat, "pa.DataType"]]], default=None
-            Provides per column name overrides for `arrow_string_format_default`
+            Per-column overrides for `arrow_string_format_default`. Keys are column names.
 
         Returns
         -------
@@ -2063,10 +2063,9 @@ def read(
         1       6
         2       7
 
-        Passing an output_format can change the resulting dataframe type. E.g. we can use the experimental arrow output
-        format:
+        Passing an output_format can change the resulting dataframe type. For example, to return a PyArrow table:
 
-        >>> lib.read("symbol", output_format="EXPERIMENTAL_ARROW").data
+        >>> lib.read("symbol", output_format="PYARROW").data
         pyarrow.Table
         column: int64
         ----
@@ -2128,19 +2127,20 @@ def read_batch(
             documentation on `LazyDataFrameCollection` for more details.
 
         output_format: Optional[Union[OutputFormat, str]], default=None
-            Controls the output format of the result dataframes.
-            For more information see documentation of `Arctic.__init__`.
-            If `None` uses the default output format from the `Library` instance.
+            Output format for the returned dataframes.
+            If `None`, uses the output format from the `Library` instance.
+            See `OutputFormat` documentation for details on available formats.
 
         arrow_string_format_default: Optional[Union[ArrowOutputStringFormat, "pa.DataType"]], default=None
-            If using `output_format=EXPERIMENTAL_ARROW` it sets the output format of string columns for arrow.
-            See documentation of `ArrowOutputStringFormat` for more information on the different options.
-            It serves as the default for the entire batch. The string format settings inside the `ReadRequest`s will
-            override this batch level setting.
+            String column format when using `PYARROW` or `POLARS` output formats.
+            Serves as the default for the entire batch. String format settings in individual `ReadRequest` objects
+            override this batch-level setting.
+            If `None`, uses the `arrow_string_format_default` from the `Library` instance.
+            See `ArrowOutputStringFormat` documentation for details on available string formats.
 
-        arrow_string_format_per_column: Optional[Dict[str, Union[ArrowOutputStringFormat, "pa.DataType"]]], default=None,
-            Provides per column name overrides for `arrow_string_format_default`. It is only applied to symbols which
-            don't have a `arrow_string_format_per_column` set in their `ReadRequest`.
+        arrow_string_format_per_column: Optional[Dict[str, Union[ArrowOutputStringFormat, "pa.DataType"]]], default=None
+            Per-column overrides for `arrow_string_format_default`. Keys are column names.
+            Only applied to symbols that don't have `arrow_string_format_per_column` set in their `ReadRequest`.
 
         Returns
         -------
@@ -2309,17 +2309,17 @@ def read_batch_and_join(
             individual dataframes, and will be applied to the joined data.
 
         output_format: Optional[Union[OutputFormat, str]], default=None
-            Controls the output format of the result dataframe.
-            For more information see documentation of `Arctic.__init__`.
-            If `None` uses the default output format from the `Library` instance.
+            Output format for the returned joined dataframe.
+            If `None`, uses the output format from the `Library` instance.
+            See `OutputFormat` documentation for details on available formats.
 
         arrow_string_format_default: Optional[Union[ArrowOutputStringFormat, "pa.DataType"]], default=None
-            If using `output_format=EXPERIMENTAL_ARROW` it sets the output format of string columns for arrow.
-            See documentation of `ArrowOutputStringFormat` for more information on the different options.
-            If `None` uses the default arrow_string_format from the `Library` instance.
+            String column format when using `PYARROW` or `POLARS` output formats.
+            If `None`, uses the `arrow_string_format_default` from the `Library` instance.
+            See `ArrowOutputStringFormat` documentation for details on available string formats.
 
         arrow_string_format_per_column: Optional[Dict[str, Union[ArrowOutputStringFormat, "pa.DataType"]]], default=None
-            Provides per column name overrides for `arrow_string_format_default`
+            Per-column overrides for `arrow_string_format_default`. Keys are column names.
 
         Returns
         -------

python/arcticdb/version_store/library.py

@@ -41,7 +41,7 @@ def setup_cache(self):
         self.logger.info(f"SETUP_CACHE TIME: {time.time() - start}")
 
     def _setup_cache(self):
-        self.ac = Arctic(self.connection_string, output_format=OutputFormat.EXPERIMENTAL_ARROW)
+        self.ac = Arctic(self.connection_string, output_format=OutputFormat.PYARROW)
         num_rows, date_ranges = self.params
         num_cols = 9  # 10 including the index column
         self.ac.delete_library(self.lib_name_prewritten)
@@ -63,7 +63,7 @@ def teardown(self, rows, date_range):
         del self.ac
 
     def setup(self, rows, date_range):
-        self.ac = Arctic(self.connection_string, output_format=OutputFormat.EXPERIMENTAL_ARROW)
+        self.ac = Arctic(self.connection_string, output_format=OutputFormat.PYARROW)
         self.lib = self.ac.get_library(self.lib_name_prewritten)
         self.lib._nvs._set_allow_arrow_input()
         if date_range is None:
@@ -126,7 +126,7 @@ def _generate_table(self, num_rows, num_cols, unique_string_count):
         return pa.Table.from_pandas(df)
 
     def _setup_cache(self):
-        self.ac = Arctic(self.connection_string, output_format=OutputFormat.EXPERIMENTAL_ARROW)
+        self.ac = Arctic(self.connection_string, output_format=OutputFormat.PYARROW)
         num_rows, date_ranges, unique_string_counts, arrow_string_format = self.params
         self.ac.delete_library(self.lib_name_prewritten)
         self.ac.create_library(self.lib_name_prewritten)
@@ -145,7 +145,7 @@ def teardown(self, rows, date_range, unique_string_count, arrow_string_format):
         del self.ac
 
     def setup(self, rows, date_range, unique_string_count, arrow_string_format):
-        self.ac = Arctic(self.connection_string, output_format=OutputFormat.EXPERIMENTAL_ARROW)
+        self.ac = Arctic(self.connection_string, output_format=OutputFormat.PYARROW)
         self.lib = self.ac.get_library(self.lib_name_prewritten)
         self.lib._nvs._set_allow_arrow_input()
         if date_range is None: