From 397fcda29555197bc2ac090ae26f9ed8e6ca3087 Mon Sep 17 00:00:00 2001 From: Tobias Windisch Date: Fri, 5 Jan 2024 19:53:53 +0100 Subject: [PATCH] mention pyspark licence and add comment about delta.io Signed-off-by: Tobias Windisch --- README.md | 1 + docs/source/conf.py | 2 +- docs/source/decorators.rst | 6 ++++++ docs/source/tutorials/spark.rst | 5 +++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 144a301..26bb437 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ Please have a look at our [contribution guide](https://boschglobal.github.io/lui | [pipdeptree](https://github.com/tox-dev/pipdeptree)|[MIT License](https://github.com/tox-dev/pipdeptree/blob/main/LICENSE) | Dependency | | [requirements-parser](https://github.com/madpah/requirements-parser)|[Apache License 2.0](https://github.com/madpah/requirements-parser/blob/master/LICENSE)| Dependency | | [pyarrow](https://github.com/apache/arrow)|[Apache License 2.0](https://github.com/apache/arrow/blob/master/LICENSE.txt)| Dependency | +| [spark](https://github.com/apache/spark)|[Apache License 2.0](https://github.com/apache/spark/blob/master/LICENSE)| Dependency | ## Development dependency diff --git a/docs/source/conf.py b/docs/source/conf.py index 1519e38..b38af88 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -93,7 +93,7 @@ intersphinx_mapping = { 'sphinx': ('http://www.sphinx-doc.org/en/master', None), 'python': ('https://docs.python.org/' + python_version, None), - 'matplotlib': ('https://matplotlib.org', None), + 'matplotlib': ('https://matplotlib.org/stable', None), 'numpy': ('https://numpy.org/doc/stable/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable', None), } diff --git a/docs/source/decorators.rst b/docs/source/decorators.rst index d85f137..7c0fecc 100644 --- a/docs/source/decorators.rst +++ b/docs/source/decorators.rst @@ -129,6 +129,12 @@ cloud storages, we provide input decorators that render the usage of See :ref:`pyspark` for more details on how to implement pipelines using these inputs. + +.. note:: + + Make sure that the `delta-spark` extension is installed into your + spark cluster. See more `here `_. + Directory structure ------------------- The following decorators denote the directory in the project directory diff --git a/docs/source/tutorials/spark.rst b/docs/source/tutorials/spark.rst index d2b3c19..92a967f 100644 --- a/docs/source/tutorials/spark.rst +++ b/docs/source/tutorials/spark.rst @@ -72,6 +72,11 @@ however, persists its output into a pickle file, which requires :py:mod:`luisy` to serialize all the data to a :py:mod:`pandas.DataFrame` beforehand. +.. note:: + + Make sure that the `delta-spark` extension is installed into your + spark cluster. See more `here `_. + Running a pipeline ------------------