chore(dev-deps): support pyspark in nix

ibis-project · Aug 2, 2024 · f8bea7e · f8bea7e
1 parent 2d3d719
commit f8bea7e
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/flake.nix b/flake.nix
@@ -65,6 +65,7 @@
 
         # necessary for quarto and quartodoc
         export PYTHONPATH=''${PWD}''${PYTHONPATH:+:}''${PYTHONPATH}:''${PWD}/docs
+        export PYSPARK_PYTHON="$(which python)"
       '';
 
       preCommitDeps = with pkgs; [

diff --git a/nix/ibis.nix b/nix/ibis.nix
@@ -7,6 +7,8 @@
 , ibisTestingData
 }:
 let
+  # pyspark could be added here, but it doesn't handle parallel test execution
+  # well and serially it takes on the order of 7-8 minutes to execute serially
   backends = [ "datafusion" "duckdb" "pandas" "polars" "sqlite" ]
     # dask version has a show-stopping bug for Python >=3.11
     ++ lib.optionals (python3.pythonOlder "3.11") [ "dask" ];