feat: support pyspark 3 (via a databricks.koalas stub)

Author: percevalw

changelog.md

@@ -2,9 +2,10 @@
 
 ## Unreleased
 
-### Changed
+### Added
 
 - Support for pyarrow > 0.17.0
+- Support for Python 3.7 to 3.10 (3.11 or higher is not tested)
 - Support for pyspark 3 (to force pyspark 2, use `pip install eds-scikit[spark2]`)
 
 ### Fixed

eds_scikit/biology/viz/aggregate.py

@@ -135,8 +135,6 @@ def aggregate_concepts_set(
 
         # Extract concept-set
         measurement_std_filtered = get_measurement_std(measurement_valid, src_to_std)
-        if is_koalas(measurement_std_filtered):
-            measurement_std_filtered.spark.cache()
         measurement_std_filtered = measurement_std_filtered.drop(
             columns="source_concept_id"
         )

tests/conftest.py

@@ -80,7 +80,6 @@ def spark_session(pytestconfig, tmpdir_factory):
         print("!! Creating spark session !!")
 
         from pyspark import SparkConf
-        from pyspark import __version__ as pyspark_version
 
         temp_warehouse_dir = tmpdir_factory.mktemp("spark")
         conf = (
@@ -100,17 +99,8 @@ def spark_session(pytestconfig, tmpdir_factory):
                 f"jdbc:derby:;databaseName={temp_warehouse_dir}/metastore_db;create=true",
             )
             .set("spark.executor.cores", 1)
-            .set("spark.driver.memory", "2g")
         )
 
-        if pyspark_version < "3":
-
-            # used to overwrite hive tables
-            conf = conf.set(
-                "spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation",
-                "true",
-            )
-
         session, _, _ = improve_performances(to_add_conf=list(conf.getAll()))
 
         # session is ready

tests/test_biology.py

@@ -14,7 +14,10 @@ def tmp_biology_dir(tmp_path_factory):
 
 @pytest.fixture
 def data():
-    return load_biology_data(seed=42)
+    return load_biology_data(
+        seed=42,
+        mean_measurement=500,
+    )
 
 
 @pytest.fixture
@@ -73,6 +76,7 @@ def test_biology_summary(data, concepts_sets, module, tmp_biology_dir):
         limit_count=("AnaBio", 500),
         stats_only=True,
         save_folder_path=tmp_biology_dir,
+        pd_limit_size=0,
     )