feat: 5.1 testing

backend/layers/processing/schema_migration.py

@@ -64,11 +64,36 @@ def gather_collections(self, auto_publish: bool) -> Tuple[Dict[str, str], Dict[s
 
         has_revision = set()
         # iterates over unpublished collections first, so published versions are skipped if there is an active revision
+
+        # temp: filter for Collections containing visium and slide-seq-v2 datasets
+        # TODO remove commented sectection bellow
+        # COLLECTIONS = [
+        #     "e68d20e4-5c51-4f6c-b599-30a08858c5ce",
+        #     "1dbbd47f-80be-4048-8516-c9fa50cb4890",
+        #     "d1c3e6e4-950e-4a5f-83b4-8de13153299a",
+        #     "362505ba-aff6-4589-a407-667a9e18bf8a",
+        #     "8dac14c9-e100-45c7-8ace-051ac2b7737a",
+        #     "19b54e58-8e71-497b-ad03-aa9a6ca52ce0",
+        #     "8fcfd5ec-e671-4836-9a96-52110e11db32",
+        #     "790b6cbd-2cec-4782-8173-08d53e6afc43",
+        #     "e0612e2e-0867-4773-bff9-b5e08b2a77e7",
+        #     "d74b6979-efba-47cd-990a-9d80ccf29055",
+        #     "02b01703-bf1b-48de-b99a-23bef8cccc81",
+        #     "8e880741-bf9a-4c8e-9227-934204631d2a",
+        #     "a96133de-e951-4e2d-ace6-59db8b3bfb1d",
+        #     "21bbfaec-6958-46bc-b1cd-1535752f6304",
+        #     "2d69ef57-864d-4f58-93f5-4fbd6fab06a6",
+        #     "7c1fbbae-5f69-4e3e-950d-d819466aecb2",
+        #     "0aab20b3-c30c-4606-bd2e-d20dae739c45",
+        # ]
+
         for collection in self.fetch_collections():
             _resp = {}
             if collection.is_published() and collection.collection_id.id in has_revision:
                 continue
-
+            # if collection.collection_id.id not in COLLECTIONS:
+            #     continue
+            # TODO: remove commented section
             if collection.is_published():
                 # published collection without an active revision
                 _resp["can_publish"] = str(True)

python_dependencies/processing/requirements.txt

@@ -2,12 +2,13 @@ anndata==0.8.0
 awscli
 boto3>=1.11.17
 botocore>=1.14.17
-cellxgene-schema
+# install the cellxgene-schema version from latest commit on the main branch from github
+git+https://github.com/chanzuckerberg/single-cell-curation.git@main#egg=cellxgene-schema&subdirectory=cellxgene_schema_cli
 dataclasses-json
 ddtrace==2.1.4
-numba==0.56.2
-numpy==1.23.2
-pandas==1.4.4
+numba
+numpy
+pandas
 psutil>=5.9.0
 psycopg2-binary==2.*
 pyarrow>=1.0
@@ -18,7 +19,7 @@ requests>=2.22.0
 rpy2==3.5.16
 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability
 s3fs==0.4.2
-scanpy==1.9.3
+scanpy
 SQLAlchemy==2.*
 tenacity
 tiledb==0.25.0  # Portal's tiledb version should always be the same or older than Explorer's

tests/unit/processing/test_h5ad_data_file.py

@@ -171,7 +171,7 @@ def test__slash_in_attribute_name(self):
 
         col_name = "fo/o"
 
-        attrs = [tiledb.Attr(name=col_name, dtype=np.int)]
+        attrs = [tiledb.Attr(name=col_name, dtype=int)]
         domain = tiledb.Domain(tiledb.Dim(domain=(0, 99), tile=100, dtype=np.uint32))
         schema = tiledb.ArraySchema(
             domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
@@ -181,7 +181,7 @@ def test__slash_in_attribute_name(self):
         try:
             with tiledb.open("foo", mode="w") as A:
                 value = dict()
-                value[col_name] = np.zeros((100,), dtype=np.int)
+                value[col_name] = np.zeros((100,), dtype=int)
                 A[:] = value  # if there's a regression, this statement will throw a TileDBError
                 # if we get here we're good
         finally:

tests/unit/processing/test_spatial_assets_utils.py

@@ -248,16 +248,17 @@ def test__upload_assets_failure(spatial_processor, asset_folder, dataset_version
     mock_upload.assert_called_once_with(asset_folder, expected_s3_uri)
 
 
-def test__create_deep_zoom_assets(spatial_processor, cxg_container, valid_spatial_data, dataset_version_id, mocker):
+def test__create_deep_zoom_assets(
+    spatial_processor, cxg_container, valid_spatial_data, dataset_version_id, mocker, tmpdir
+):
     mock_fetch_image = mocker.patch.object(spatial_processor, "_fetch_image")
     mock_process_and_flip_image = mocker.patch.object(spatial_processor, "_process_and_flip_image")
     mock_generate_deep_zoom_assets = mocker.patch.object(spatial_processor, "_generate_deep_zoom_assets")
     mock_upload_assets = mocker.patch.object(spatial_processor, "_upload_assets")
 
     # mock the TemporaryDirectory context manager
     mock_temp_dir = mocker.patch("tempfile.TemporaryDirectory")
-    temp_dir_name = "/mock/temp/dir"
-    mock_temp_dir.return_value.__enter__.return_value = temp_dir_name
+    mock_temp_dir.return_value.__enter__.return_value = tmpdir
 
     # mock return values for the internal methods
     mock_fetch_image.return_value = (np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8), "fullres")
@@ -266,7 +267,7 @@ def test__create_deep_zoom_assets(spatial_processor, cxg_container, valid_spatia
     # call the method under test
     spatial_processor.create_deep_zoom_assets(cxg_container, valid_spatial_data, dataset_version_id)
 
-    assets_folder = os.path.join(temp_dir_name, cxg_container.replace(".cxg", ""))
+    assets_folder = os.path.join(tmpdir, cxg_container.replace(".cxg", ""))
 
     # assertions to ensure each step is called
     mock_fetch_image.assert_called_once_with(valid_spatial_data)