reorganizing tests to run a suite of tests for each LTS DBR version since Spark 3.0

Tristan Nixon · Tristan Nixon · commit 55cf05aecf13 · 2023-11-02T15:59:50.000-07:00
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -0,0 +1,6 @@
+sphinx-autobuild==2021.3.14
+sphinx-copybutton==0.5.1
+Sphinx==4.5.0
+sphinx-design==0.2.0
+sphinx-panels==0.6.0
+furo==2022.9.29
diff --git a/python/requirements.txt b/python/requirements.txt
diff --git a/python/requirements/dbr104.txt b/python/requirements/dbr104.txt
@@ -0,0 +1,10 @@
+ipython==7.22.0
+numpy==1.20.1
+pandas==1.2.4
+pyarrow==4.0.0
+python-dateutil==2.8.1
+pytz==2020.5
+scipy==1.6.2
+six==1.15.0
+wheel==0.36.2
+
diff --git a/python/requirements/dbr113.txt b/python/requirements/dbr113.txt
@@ -0,0 +1,9 @@
+ipython==7.32.0
+numpy==1.20.3
+pandas==1.3.4
+pyarrow==7.0.0
+python-dateutil==2.8.2
+pytz==2021.3
+scipy==1.7.1
+six==1.16.0
+wheel==0.37.0
diff --git a/python/requirements/dbr122.txt b/python/requirements/dbr122.txt
@@ -0,0 +1,9 @@
+ipython==8.5.0
+numpy==1.21.5
+pandas==1.4.2
+pyarrow==7.0.0
+python-dateutil==2.8.2
+pytz==2021.3
+scipy==1.7.3
+six==1.16.0
+wheel==0.37.0
diff --git a/python/requirements/dbr133.txt b/python/requirements/dbr133.txt
@@ -0,0 +1,9 @@
+ipython==8.10.0
+numpy==1.21.5
+pandas==1.4.4
+pyarrow==8.0.0
+python-dateutil==2.8.2
+pytz==2022.1
+scipy==1.9.1
+six==1.16.0
+wheel==0.37.1
diff --git a/python/requirements/dbr73.txt b/python/requirements/dbr73.txt
@@ -0,0 +1,10 @@
+ipython==7.12.0
+numpy==1.18.1
+pandas==1.0.1
+pyarrow==1.0.1
+python-dateutil==2.8.1
+pytz==2019.3
+scipy==1.4.1
+six==1.14.0
+wheel==0.34.2
+
diff --git a/python/requirements/dbr91.txt b/python/requirements/dbr91.txt
@@ -0,0 +1,10 @@
+ipython==7.22.0
+numpy==1.19.2
+pandas==1.2.4
+pyarrow==4.0.0
+python-dateutil==2.8.1
+pytz==2020.5
+scipy==1.6.2
+six==1.15.0
+wheel==0.36.2
+
diff --git a/python/tox.ini b/python/tox.ini
@@ -2,7 +2,6 @@
 requires =
     tox>4,<5
     virtualenv>20,<21
-    wheel>=0.38,<1
 isolated_build = true
 envlist =
     format
@@ -11,26 +10,34 @@ envlist =
     build-dist
     ; Mirror Supported LTS DBR versions here: https://docs.databricks.com/release-notes/runtime/
     ; Use correct PySpark version based on Python version present in env name
-    py37-pyspark300,
-    py38-pyspark{312,321},
-    py39-pyspark{330,332}
+    dbr{73,91,104,113,122,133}
 skip_missing_interpreters = true
 
-
 [testenv]
 description = run the tests under {envname}
 package = wheel
 wheel_build_env = .pkg
 setenv =
     COVERAGE_FILE = .coverage.{envname}
+basepython =
+    dbr133: py310
+    dbr122: py39
+    dbr113: py39
+    dbr104: py38
+    dbr91: py38
+    dbr73: py37
 deps =
-    pyspark300: pyspark==3.0.0
-    pyspark312: pyspark==3.1.2
-    pyspark321: pyspark==3.2.1
-    pyspark330: pyspark==3.3.0
-    pyspark332: pyspark==3.3.2
+    dbr133: pyspark==3.4.1
+    dbr122: pyspark==3.3.2
+    dbr113: pyspark==3.3.0
+    dbr104: pyspark==3.2.1
+    dbr91: pyspark==3.1.2
+    dbr73: pyspark==3.0.0
     coverage>=7,<8
-    -rrequirements.txt
+    chispa~=0.9.4
+    semver>=3,<4
+    jsonref>=1,<2
+    -rrequirements/{envname}.txt
 commands =
     coverage --version
     coverage run -m unittest discover -s tests -p '*_tests.py'
@@ -63,7 +70,7 @@ deps =
     mypy>=1,<2
     pandas-stubs>=2,<3
     types-pytz>=2023,<2024
-    -rrequirements.txt
+    -rrequirements/dbr133.txt
 commands =
     mypy {toxinidir}/tempo
 
