Skip to content

Commit

Permalink
can't import from private pyspark typing modules
Browse files Browse the repository at this point in the history
workaround is to copy definitions locally
  • Loading branch information
tnixon committed Jan 3, 2024
1 parent 1c18b01 commit 39ce9c7
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 2 deletions.
3 changes: 1 addition & 2 deletions python/tempo/tsdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
from pyspark.sql.column import Column
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.types import DataType, StructType
from pyspark.sql._typing import ColumnOrName
from pyspark.sql.pandas._typing import PandasMapIterFunction, PandasGroupedMapFunction
from pyspark.sql.window import Window, WindowSpec

import tempo.interpol as t_interpolation
Expand All @@ -26,6 +24,7 @@
import tempo.utils as t_utils
from tempo.intervals import IntervalsDF
from tempo.tsschema import CompositeTSIndex, TSIndex, TSSchema, WindowBuilder
from tempo.typing import ColumnOrName, PandasMapIterFunction, PandasGroupedMapFunction

logger = logging.getLogger(__name__)

Expand Down
20 changes: 20 additions & 0 deletions python/tempo/typing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
from typing import Union, Callable, Iterable, Any

from pyspark.sql import Column

from pandas.core.frame import DataFrame as PandasDataFrame

from pyspark.sql.pandas._typing import PandasMapIterFunction, PandasGroupedMapFunction

# These definitions were copied from private pypark modules:
# - pyspark.sql._typing
# - pyspark.sql.pandas._typing

ColumnOrName = Union[Column, str]

PandasMapIterFunction = Callable[[Iterable[PandasDataFrame]], Iterable[PandasDataFrame]]

PandasGroupedMapFunction = Union[
Callable[[PandasDataFrame], PandasDataFrame],
Callable[[Any, PandasDataFrame], PandasDataFrame],
]

0 comments on commit 39ce9c7

Please sign in to comment.