diff --git a/backend/layers/processing/schema_migration.py b/backend/layers/processing/schema_migration.py index fe0f8bda71d47..c888a0538dec9 100644 --- a/backend/layers/processing/schema_migration.py +++ b/backend/layers/processing/schema_migration.py @@ -64,11 +64,36 @@ def gather_collections(self, auto_publish: bool) -> Tuple[Dict[str, str], Dict[s has_revision = set() # iterates over unpublished collections first, so published versions are skipped if there is an active revision + + # temp: filter for Collections containing visium and slide-seq-v2 datasets + # TODO remove commented sectection bellow + # COLLECTIONS = [ + # "e68d20e4-5c51-4f6c-b599-30a08858c5ce", + # "1dbbd47f-80be-4048-8516-c9fa50cb4890", + # "d1c3e6e4-950e-4a5f-83b4-8de13153299a", + # "362505ba-aff6-4589-a407-667a9e18bf8a", + # "8dac14c9-e100-45c7-8ace-051ac2b7737a", + # "19b54e58-8e71-497b-ad03-aa9a6ca52ce0", + # "8fcfd5ec-e671-4836-9a96-52110e11db32", + # "790b6cbd-2cec-4782-8173-08d53e6afc43", + # "e0612e2e-0867-4773-bff9-b5e08b2a77e7", + # "d74b6979-efba-47cd-990a-9d80ccf29055", + # "02b01703-bf1b-48de-b99a-23bef8cccc81", + # "8e880741-bf9a-4c8e-9227-934204631d2a", + # "a96133de-e951-4e2d-ace6-59db8b3bfb1d", + # "21bbfaec-6958-46bc-b1cd-1535752f6304", + # "2d69ef57-864d-4f58-93f5-4fbd6fab06a6", + # "7c1fbbae-5f69-4e3e-950d-d819466aecb2", + # "0aab20b3-c30c-4606-bd2e-d20dae739c45", + # ] + for collection in self.fetch_collections(): _resp = {} if collection.is_published() and collection.collection_id.id in has_revision: continue - + # if collection.collection_id.id not in COLLECTIONS: + # continue + # TODO: remove commented section if collection.is_published(): # published collection without an active revision _resp["can_publish"] = str(True) diff --git a/python_dependencies/processing/requirements.txt b/python_dependencies/processing/requirements.txt index 5396acfd9dc69..31e8a22bf0f5b 100644 --- a/python_dependencies/processing/requirements.txt +++ b/python_dependencies/processing/requirements.txt @@ -2,12 +2,13 @@ anndata==0.8.0 awscli boto3>=1.11.17 botocore>=1.14.17 -cellxgene-schema +# install the cellxgene-schema version from latest commit on the main branch from github +git+https://github.com/chanzuckerberg/single-cell-curation.git@main#egg=cellxgene-schema&subdirectory=cellxgene_schema_cli dataclasses-json ddtrace==2.1.4 -numba==0.56.2 -numpy==1.23.2 -pandas==1.4.4 +numba +numpy +pandas psutil>=5.9.0 psycopg2-binary==2.* pyarrow>=1.0 @@ -18,7 +19,7 @@ requests>=2.22.0 rpy2==3.5.16 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability s3fs==0.4.2 -scanpy==1.9.3 +scanpy SQLAlchemy==2.* tenacity tiledb==0.25.0 # Portal's tiledb version should always be the same or older than Explorer's diff --git a/tests/unit/processing/test_h5ad_data_file.py b/tests/unit/processing/test_h5ad_data_file.py index f3d01c06ec47e..7fbeb0e7bcceb 100644 --- a/tests/unit/processing/test_h5ad_data_file.py +++ b/tests/unit/processing/test_h5ad_data_file.py @@ -171,7 +171,7 @@ def test__slash_in_attribute_name(self): col_name = "fo/o" - attrs = [tiledb.Attr(name=col_name, dtype=np.int)] + attrs = [tiledb.Attr(name=col_name, dtype=int)] domain = tiledb.Domain(tiledb.Dim(domain=(0, 99), tile=100, dtype=np.uint32)) schema = tiledb.ArraySchema( domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major" @@ -181,7 +181,7 @@ def test__slash_in_attribute_name(self): try: with tiledb.open("foo", mode="w") as A: value = dict() - value[col_name] = np.zeros((100,), dtype=np.int) + value[col_name] = np.zeros((100,), dtype=int) A[:] = value # if there's a regression, this statement will throw a TileDBError # if we get here we're good finally: diff --git a/tests/unit/processing/test_spatial_assets_utils.py b/tests/unit/processing/test_spatial_assets_utils.py index 2a69f8ff012f7..cc40bb43dface 100644 --- a/tests/unit/processing/test_spatial_assets_utils.py +++ b/tests/unit/processing/test_spatial_assets_utils.py @@ -248,7 +248,9 @@ def test__upload_assets_failure(spatial_processor, asset_folder, dataset_version mock_upload.assert_called_once_with(asset_folder, expected_s3_uri) -def test__create_deep_zoom_assets(spatial_processor, cxg_container, valid_spatial_data, dataset_version_id, mocker): +def test__create_deep_zoom_assets( + spatial_processor, cxg_container, valid_spatial_data, dataset_version_id, mocker, tmpdir +): mock_fetch_image = mocker.patch.object(spatial_processor, "_fetch_image") mock_process_and_flip_image = mocker.patch.object(spatial_processor, "_process_and_flip_image") mock_generate_deep_zoom_assets = mocker.patch.object(spatial_processor, "_generate_deep_zoom_assets") @@ -256,8 +258,7 @@ def test__create_deep_zoom_assets(spatial_processor, cxg_container, valid_spatia # mock the TemporaryDirectory context manager mock_temp_dir = mocker.patch("tempfile.TemporaryDirectory") - temp_dir_name = "/mock/temp/dir" - mock_temp_dir.return_value.__enter__.return_value = temp_dir_name + mock_temp_dir.return_value.__enter__.return_value = tmpdir # mock return values for the internal methods mock_fetch_image.return_value = (np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8), "fullres") @@ -266,7 +267,7 @@ def test__create_deep_zoom_assets(spatial_processor, cxg_container, valid_spatia # call the method under test spatial_processor.create_deep_zoom_assets(cxg_container, valid_spatial_data, dataset_version_id) - assets_folder = os.path.join(temp_dir_name, cxg_container.replace(".cxg", "")) + assets_folder = os.path.join(tmpdir, cxg_container.replace(".cxg", "")) # assertions to ensure each step is called mock_fetch_image.assert_called_once_with(valid_spatial_data)