Release 0.4.0

Co-authored-by: Michał Bartoszkiewicz <embe@pathway.com>
Co-authored-by: Jan Chorowski <janek@pathway.com>
Co-authored-by: Xavier Gendre <xavier@pathway.com>
Co-authored-by: Adrian Kosowski <adrian@pathway.com>
Co-authored-by: Jakub Kowalski <kuba@pathway.com>
Co-authored-by: Sergey Kulik <sergey@pathway.com>
Co-authored-by: Mateusz Lewandowski <mateusz@pathway.com>
Co-authored-by: Mohamed Malhou <mohamed@pathway.com>
Co-authored-by: Krzysztof Nowicki <krzysiek@pathway.com>
Co-authored-by: Richard Pelgrim <richard.pelgrim@pathway.com>
Co-authored-by: Kamil Piechowiak <kamil@pathway.com>
Co-authored-by: Paweł Podhajski <pawel.podhajski@pathway.com>
Co-authored-by: Olivier Ruas <olivier@pathway.com>
Co-authored-by: Przemysław Uznański <przemek@pathway.com>
Co-authored-by: Sebastian Włudzik <sebastian.wludzik@pathway.com>
GitOrigin-RevId: b78921d6f077931456006b82bc4e68ae9f5ac24c

.github/workflows/release.yml

@@ -145,6 +145,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Create dir for wheels
+        run: |
+          mkdir wheels
+            
       - uses: actions/download-artifact@master
         with:
           name: pathway-x86-x64
@@ -157,7 +161,7 @@ jobs:
           rm -rf "${ENV_NAME}"
           python -m venv "${ENV_NAME}"
           source "${ENV_NAME}/bin/activate"
-          WHEEL=(public/pathway/target/wheels/pathway-*.whl)
+          WHEEL=(./wheels/pathway-*.whl)
           pip install --prefer-binary "${WHEEL}[tests]"
           # --confcutdir anything below to avoid picking REPO_TOP_DIR/conftest.py
           python -m pytest --confcutdir "${ENV_NAME}" --doctest-modules --pyargs pathway
@@ -177,7 +181,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
     steps:
-      - name: create dir for wheels
+      - name: Create dir for wheels
         run: |
           mkdir wheels
 
@@ -202,7 +206,7 @@ jobs:
           rm -rf "${ENV_NAME}"
           python"${{ matrix.python-version }}" -m venv "${ENV_NAME}"
           source "${ENV_NAME}/bin/activate"
-          WHEEL=(public/pathway/target/wheels/pathway-*.whl)
+          WHEEL=(./wheels/pathway-*.whl)
           pip install --prefer-binary "${WHEEL}[tests]"
           # --confcutdir anything below to avoid picking REPO_TOP_DIR/conftest.py
           python -m pytest --confcutdir "${ENV_NAME}" --doctest-modules --pyargs pathway
@@ -235,7 +239,7 @@ jobs:
           aws-region: ${{ secrets.ARTIFACT_AWS_REGION }}
           role-duration-seconds: 1200
 
-      - name: create dir for wheels
+      - name: Create dir for wheels
         run: |
           mkdir wheels
 

CHANGELOG.md

@@ -6,6 +6,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
+## [0.4.0] - 2023-09-21
+
+### Added
+- Support for JSON data format, including `pw.Json` type.
+- Methods `as_int()`, `as_float()`, `as_str()`, `as_bool()` to convert values from `Json`.
+
+### Changed
+- Method `get()` and `[]` to support accessing elements in Jsons.
+- Function `pw.assert_table_has_schema` for writing asserts checking, whether given table has the same schema as the one that is given as an argument.
+- **BREAKING**: `ix` and `ix_ref` operations are now standalone transformations of `pw.Table` into `pw.Table`. Most of the usages remain the same, but sometimes user needs to provide a context (when e.g. using them inside `join` or `groupby` operations). `ix` and `ix_ref` are temporarily broken inside temporal joins.
+
+### Fixed
+- Fixed a bug where new-style optional types (e.g. `int | None`) were translated to `Any` dtype.
+
 ## [0.3.4] - 2023-09-18
 
 ### Fixed

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pathway"
-version = "0.3.4"
+version = "0.4.0"
 edition = "2021"
 publish = false
 rust-version = "1.71.0"
@@ -22,7 +22,7 @@ arcstr = { version = "1.1.5", default-features = false, features = ["serde", "st
 base32 = "0.4.0"
 bincode = "1.3.3"
 cfg-if = "1.0.0"
-chrono = { version = "0.4.30", features = ["std", "clock"], default-features = false }
+chrono = { version = "0.4.31", features = ["std", "clock"], default-features = false }
 chrono-tz = "0.8.3"
 crossbeam-channel = "0.5.8"
 csv = "1.2.2"

python/pathway/__init__.py

@@ -31,6 +31,7 @@
     Joinable,
     JoinMode,
     JoinResult,
+    Json,
     MonitoringLevel,
     Pointer,
     Schema,
@@ -42,6 +43,7 @@
     apply,
     apply_async,
     apply_with_type,
+    assert_table_has_schema,
     asynchronous,
     attribute,
     cast,
@@ -148,6 +150,8 @@
     "DURATION",
     "unwrap",
     "SchemaProperties",
+    "assert_table_has_schema",
+    "Json",
 ]
 
 

python/pathway/debug/__init__.py

@@ -10,7 +10,7 @@
 
 import pandas as pd
 
-from pathway.internals import api, parse_graph
+from pathway.internals import Json, api, parse_graph
 from pathway.internals.datasource import PandasDataSource
 from pathway.internals.decorators import table_from_datasource
 from pathway.internals.graph_runner import GraphRunner
@@ -37,7 +37,10 @@ def table_to_dicts(table: Table):
 @functools.total_ordering
 class _NoneAwareComparisonWrapper:
     def __init__(self, inner):
-        self.inner = inner
+        if isinstance(inner, dict | Json):
+            self.inner = str(inner)
+        else:
+            self.inner = inner
 
     def __eq__(self, other):
         if not isinstance(other, _NoneAwareComparisonWrapper):

python/pathway/dt.py

@@ -7,6 +7,7 @@
     DURATION,
     FLOAT,
     INT,
+    JSON,
     POINTER,
     STR,
     Array,
@@ -26,6 +27,7 @@
     "DURATION",
     "FLOAT",
     "INT",
+    "JSON",
     "POINTER",
     "STR",
     "Array",

python/pathway/engine.pyi

@@ -29,6 +29,7 @@ class PathwayType(Enum):
     DATE_TIME_UTC: PathwayType
     DURATION: PathwayType
     ARRAY: PathwayType
+    JSON: PathwayType
 
 class ConnectorMode(Enum):
     STATIC: ConnectorMode
@@ -52,15 +53,15 @@ class Trace:
 
 @dataclasses.dataclass(frozen=True)
 class EvalProperties:
-    trace: Optional[Trace] = None
     dtype: Optional[DType] = None
-    append_only: Optional[bool] = False
+    trace: Optional[Trace] = None
+    append_only: bool = False
 
 @dataclasses.dataclass(frozen=True)
 class ConnectorProperties:
     commit_duration_ms: Optional[int] = None
     unsafe_trusted_ids: Optional[bool] = False
-    append_only: Optional[bool] = False
+    append_only: bool = False
 
 class Column:
     """A Column holds data and conceptually is a Dict[Universe elems, dt]
@@ -165,6 +166,9 @@ class Expression:
     def cast_optional(
         expr: Expression, source_type: PathwayType, target_type: PathwayType
     ) -> Optional[Expression]: ...
+    def convert_optional(
+        expr: Expression, source_type: PathwayType, target_type: PathwayType
+    ) -> Optional[Expression]: ...
     @staticmethod
     def if_else(if_: Expression, then: Expression, else_: Expression) -> Expression: ...
     @staticmethod
@@ -276,6 +280,12 @@ class Expression:
         expr: Expression, index: Expression
     ) -> Expression: ...
     @staticmethod
+    def json_get_item_checked(
+        expr: Expression, index: Expression, default: Expression
+    ) -> Expression: ...
+    @staticmethod
+    def json_get_item_unchecked(expr: Expression, index: Expression) -> Expression: ...
+    @staticmethod
     def unwrap(expr: Expression) -> Expression: ...
     @staticmethod
     def to_string(expr: Expression) -> Expression: ...
@@ -459,9 +469,6 @@ class Grouper:
     def input_column(self, column: Column) -> Column: ...
     def count_column(self) -> Column: ...
     def reducer_column(self, reducer: Reducer, column: Column) -> Column: ...
-    def reducer_ix_column(
-        self, reducer: Reducer, ixer: Ixer, input_column: Column
-    ) -> Column: ...
 
 class Concat:
     @property

python/pathway/internals/__init__.py

@@ -8,6 +8,7 @@
     apply,
     apply_async,
     apply_with_type,
+    assert_table_has_schema,
     cast,
     coalesce,
     declare_type,
@@ -31,14 +32,13 @@
 from pathway.internals.expression import (
     ColumnExpression,
     ColumnExpressionOrValue,
-    ColumnIxExpression,
     ColumnReference,
     ReducerExpression,
-    ReducerIxExpression,
 )
 from pathway.internals.groupby import GroupedJoinResult, GroupedTable
 from pathway.internals.join import FilteredJoinResult, Joinable, JoinResult
 from pathway.internals.join_mode import JoinMode
+from pathway.internals.json import Json
 from pathway.internals.monitoring import MonitoringLevel
 from pathway.internals.operator import iterate_universe
 from pathway.internals.row_transformer import ClassArg
@@ -104,8 +104,6 @@
     "run",
     "run_all",
     "numba_apply",
-    "ColumnIxExpression",
-    "ReducerIxExpression",
     "__version__",
     "universes",
     "asynchronous",
@@ -118,4 +116,6 @@
     "DURATION",
     "unwrap",
     "SchemaProperties",
+    "assert_table_has_schema",
+    "Json",
 ]

python/pathway/internals/api.py

@@ -32,7 +32,9 @@
 # XXX: engine calls return BasePointer, not Pointer
 class Pointer(BasePointer, Generic[TSchema]):
     """Pointer to row type.
+
     Example:
+
     >>> import pathway as pw
     >>> t1 = pw.debug.parse_to_table('''
     ... age | owner | pet
@@ -109,4 +111,5 @@ def denumpify(x):
     dt.DURATION: PathwayType.DURATION,
     dt.Array(): PathwayType.ARRAY,
     dt.ANY: PathwayType.ANY,
+    dt.JSON: PathwayType.JSON,
 }

python/pathway/internals/arg_handlers.py

@@ -137,7 +137,7 @@ def reduce_args_handler(self, *args, **kwargs):
 
 def select_args_handler(self, *args, **kwargs):
     for arg in args:
-        if not isinstance(arg, expr.ColumnRefOrIxExpression):
+        if not isinstance(arg, expr.ColumnReference):
             if isinstance(arg, str):
                 raise ValueError(
                     f"Expected a ColumnReference, found a string. Did you mean this.{arg} instead of {repr(arg)}?"