From d5ae91f942cf5aea07d7f606d14df886946e09c3 Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 19:31:14 -0500 Subject: [PATCH 01/12] fixed bug with inclusion of ~ in path --- test/dataset_utils.py | 17 ++++++++++------- test/test_datasets.py | 1 + test/test_dsec.py | 2 +- test/test_io.py | 4 ++-- tonic/dataset.py | 2 +- tonic/datasets/threeET_eyetracking.py | 1 + tonic/download_utils.py | 2 -- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/test/dataset_utils.py b/test/dataset_utils.py index 430d1253..ff6e41d7 100644 --- a/test/dataset_utils.py +++ b/test/dataset_utils.py @@ -2,14 +2,17 @@ import unittest from typing import Any, Dict, Union from unittest.mock import patch - +import pytest import numpy as np +import os +# Location of the files to be saved and extracted by the datasets during testing +TEST_LOCATION_ON_SYSTEM = "~/../../tmp" +TEST_LOCATION_ON_SYSTEM = os.path.expanduser(TEST_LOCATION_ON_SYSTEM) class DatasetTestCase(unittest.TestCase): DATASET_CLASS = None FEATURE_TYPES = None - _CHECK_FUNCTIONS = {"check_md5", "check_integrity", "check_exists"} _DOWNLOAD_EXTRACT_FUNCTIONS = { "download_url", @@ -17,7 +20,7 @@ class DatasetTestCase(unittest.TestCase): "extract_archive", "download_and_extract_archive", } - + def inject_fake_data( self, tmpdir: str, config: Dict[str, Any] ) -> Union[int, Dict[str, Any]]: @@ -41,8 +44,7 @@ def inject_fake_data( ) def create_dataset(self, inject_fake_data: bool = True, **kwargs: Any): - tmpdir = "/tmp/" - info = self._inject_fake_data(tmpdir) + info = self._inject_fake_data(TEST_LOCATION_ON_SYSTEM) if inject_fake_data: with patch.object(self.DATASET_CLASS, "_check_exists", return_value=True): @@ -87,12 +89,13 @@ def test_feature_types(self): else: assert type(data_piece) == feature_type + def test_num_examples(self): dataset, info = self.create_dataset() assert len(dataset) == info["n_samples"] @classmethod def setUpClass(cls): - cls.KWARGS.update({"save_to": "/tmp"}) - shutil.rmtree("/tmp/" + cls.DATASET_CLASS.__name__, ignore_errors=True) + cls.KWARGS.update({"save_to": TEST_LOCATION_ON_SYSTEM}) + shutil.rmtree(f"{TEST_LOCATION_ON_SYSTEM}/" + cls.DATASET_CLASS.__name__, ignore_errors=True) super().setUpClass() diff --git a/test/test_datasets.py b/test/test_datasets.py index 7bc4e964..c46d5052 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -33,6 +33,7 @@ class DVSGestureTestCaseTrain(dataset_utils.DatasetTestCase): def inject_fake_data(self, tmpdir): testfolder = os.path.join(tmpdir, "DVSGesture/ibmGestureTrain/user24_led") + print("TESTFOLDER ", testfolder) os.makedirs(testfolder, exist_ok=True) events, sensor_size = create_random_input(dtype=datasets.DVSGesture.dtype) events = np.lib.recfunctions.structured_to_unstructured(events) diff --git a/test/test_dsec.py b/test/test_dsec.py index bb864abe..9ff0ddb6 100644 --- a/test/test_dsec.py +++ b/test/test_dsec.py @@ -84,7 +84,7 @@ def test_target_multiselection(): assert len(targets) == 2 -# @pytest.mark.skip("Data not available from CI server...") +@pytest.mark.skip("Data not available from CI server...") def test_optical_flow(): dataset = tonic.datasets.DSEC( save_to="data", diff --git a/test/test_io.py b/test/test_io.py index 3e17adba..0c3a49ed 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -20,5 +20,5 @@ def test_read_aedat_events(): assert len(events) == 118651 -def test_read_aedat4(): - events = tonic.io.read_aedat4("test/test_data/sample.aedat4") +# def test_read_aedat4(): +# events = tonic.io.read_aedat4("test/test_data/sample.aedat4") diff --git a/tonic/dataset.py b/tonic/dataset.py index db44a3b4..4ec16d1e 100644 --- a/tonic/dataset.py +++ b/tonic/dataset.py @@ -18,7 +18,7 @@ def __init__( target_transform: Optional[Callable] = None, transforms: Optional[Callable] = None, ): - self.location_on_system = os.path.join(save_to, self.__class__.__name__) + self.location_on_system = os.path.join(os.path.expanduser(save_to), self.__class__.__name__) self.transform = transform self.target_transform = target_transform self.transforms = transforms diff --git a/tonic/datasets/threeET_eyetracking.py b/tonic/datasets/threeET_eyetracking.py index 8c50cac4..00f6a00c 100644 --- a/tonic/datasets/threeET_eyetracking.py +++ b/tonic/datasets/threeET_eyetracking.py @@ -60,6 +60,7 @@ def __init__( self.download() data_dir = os.path.join(save_to, "ThreeET_Eyetracking") + print("DATADIOR", data_dir, self.location_on_system) # Load filenames from the provided lists if split == "train": filenames = self.load_filenames(os.path.join(data_dir, "train_files.txt")) diff --git a/tonic/download_utils.py b/tonic/download_utils.py index e6135579..2cdfd459 100644 --- a/tonic/download_utils.py +++ b/tonic/download_utils.py @@ -313,12 +313,10 @@ def download_and_extract_archive( md5: Optional[str] = None, remove_finished: bool = False, ) -> None: - download_root = os.path.expanduser(download_root) if extract_root is None: extract_root = download_root if not filename: filename = os.path.basename(url) - download_url(url, download_root, filename, md5) archive = os.path.join(download_root, filename) From 2b2c25644e19a6d7e7a3664ab9d14362943602bc Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 19:33:28 -0500 Subject: [PATCH 02/12] removed testing print --- tonic/datasets/threeET_eyetracking.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tonic/datasets/threeET_eyetracking.py b/tonic/datasets/threeET_eyetracking.py index 00f6a00c..8c50cac4 100644 --- a/tonic/datasets/threeET_eyetracking.py +++ b/tonic/datasets/threeET_eyetracking.py @@ -60,7 +60,6 @@ def __init__( self.download() data_dir = os.path.join(save_to, "ThreeET_Eyetracking") - print("DATADIOR", data_dir, self.location_on_system) # Load filenames from the provided lists if split == "train": filenames = self.load_filenames(os.path.join(data_dir, "train_files.txt")) From 6a50735cfe61e51b152d4ad0893ac88b627e37ba Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 19:36:11 -0500 Subject: [PATCH 03/12] fixed commented test --- test/test_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_io.py b/test/test_io.py index 0c3a49ed..3e17adba 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -20,5 +20,5 @@ def test_read_aedat_events(): assert len(events) == 118651 -# def test_read_aedat4(): -# events = tonic.io.read_aedat4("test/test_data/sample.aedat4") +def test_read_aedat4(): + events = tonic.io.read_aedat4("test/test_data/sample.aedat4") From a072ae4bf0299bba710beede0bd8214b996edb6b Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 19:49:46 -0500 Subject: [PATCH 04/12] commmented test skip --- test/test_datasets.py | 1 - test/test_dsec.py | 2 +- tonic/download_utils.py | 6 +++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_datasets.py b/test/test_datasets.py index c46d5052..7bc4e964 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -33,7 +33,6 @@ class DVSGestureTestCaseTrain(dataset_utils.DatasetTestCase): def inject_fake_data(self, tmpdir): testfolder = os.path.join(tmpdir, "DVSGesture/ibmGestureTrain/user24_led") - print("TESTFOLDER ", testfolder) os.makedirs(testfolder, exist_ok=True) events, sensor_size = create_random_input(dtype=datasets.DVSGesture.dtype) events = np.lib.recfunctions.structured_to_unstructured(events) diff --git a/test/test_dsec.py b/test/test_dsec.py index 9ff0ddb6..bb864abe 100644 --- a/test/test_dsec.py +++ b/test/test_dsec.py @@ -84,7 +84,7 @@ def test_target_multiselection(): assert len(targets) == 2 -@pytest.mark.skip("Data not available from CI server...") +# @pytest.mark.skip("Data not available from CI server...") def test_optical_flow(): dataset = tonic.datasets.DSEC( save_to="data", diff --git a/tonic/download_utils.py b/tonic/download_utils.py index 2cdfd459..91386ede 100644 --- a/tonic/download_utils.py +++ b/tonic/download_utils.py @@ -159,9 +159,9 @@ def _extract_zip(from_path: str, to_path: str, compression: Optional[str]) -> No with zipfile.ZipFile( from_path, "r", - compression=_ZIP_COMPRESSION_MAP[compression] - if compression - else zipfile.ZIP_STORED, + compression=( + _ZIP_COMPRESSION_MAP[compression] if compression else zipfile.ZIP_STORED + ), ) as zip: zip.extractall(to_path) From 110a5bc055975b295ac434d4bf3fb15e88aad5a7 Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 19:54:41 -0500 Subject: [PATCH 05/12] formatted dataset_utils --- test/dataset_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/dataset_utils.py b/test/dataset_utils.py index ff6e41d7..4b99a194 100644 --- a/test/dataset_utils.py +++ b/test/dataset_utils.py @@ -4,12 +4,13 @@ from unittest.mock import patch import pytest import numpy as np -import os +import os # Location of the files to be saved and extracted by the datasets during testing TEST_LOCATION_ON_SYSTEM = "~/../../tmp" TEST_LOCATION_ON_SYSTEM = os.path.expanduser(TEST_LOCATION_ON_SYSTEM) + class DatasetTestCase(unittest.TestCase): DATASET_CLASS = None FEATURE_TYPES = None @@ -20,7 +21,7 @@ class DatasetTestCase(unittest.TestCase): "extract_archive", "download_and_extract_archive", } - + def inject_fake_data( self, tmpdir: str, config: Dict[str, Any] ) -> Union[int, Dict[str, Any]]: @@ -83,13 +84,12 @@ def test_feature_types(self): assert len(data) == len(self.FEATURE_TYPES) assert len(target) == len(self.TARGET_TYPES) - for (data_piece, feature_type) in zip(data, self.FEATURE_TYPES): + for data_piece, feature_type in zip(data, self.FEATURE_TYPES): if type(data_piece) == np.ndarray: assert data_piece.dtype == feature_type else: assert type(data_piece) == feature_type - def test_num_examples(self): dataset, info = self.create_dataset() assert len(dataset) == info["n_samples"] @@ -97,5 +97,8 @@ def test_num_examples(self): @classmethod def setUpClass(cls): cls.KWARGS.update({"save_to": TEST_LOCATION_ON_SYSTEM}) - shutil.rmtree(f"{TEST_LOCATION_ON_SYSTEM}/" + cls.DATASET_CLASS.__name__, ignore_errors=True) + shutil.rmtree( + f"{TEST_LOCATION_ON_SYSTEM}/" + cls.DATASET_CLASS.__name__, + ignore_errors=True, + ) super().setUpClass() From eb3adcbc31ecf0ae8949923d77bd6787779a48cd Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 20:02:36 -0500 Subject: [PATCH 06/12] added max version to torchdata --- .pre-commit-config.yaml | 6 +++++- test/torch_requirements.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcc25da4..a203d5f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,15 +4,19 @@ repos: hooks: - id: black language_version: python3 + language: python - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: - id: isort args: ["--profile", "black", "--filter-files"] + language: python + - repo: https://github.com/myint/docformatter rev: v1.6.0 hooks: - id: docformatter - args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99] \ No newline at end of file + args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99] + language: python diff --git a/test/torch_requirements.txt b/test/torch_requirements.txt index fd14302b..84d87bb6 100644 --- a/test/torch_requirements.txt +++ b/test/torch_requirements.txt @@ -2,4 +2,4 @@ torch==2.1.0 torchaudio==2.1.0 torchvision==0.16.0 -torchdata +torchdata<=0.8.0 From 7e4a03584407fbd447fbc84201975ab209a7e2b7 Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 20:03:35 -0500 Subject: [PATCH 07/12] reverted pre-commit file --- .pre-commit-config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a203d5f7..4df525f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,14 +4,12 @@ repos: hooks: - id: black language_version: python3 - language: python - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: - id: isort args: ["--profile", "black", "--filter-files"] - language: python - repo: https://github.com/myint/docformatter @@ -19,4 +17,3 @@ repos: hooks: - id: docformatter args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99] - language: python From f83df9370358360155cad2e3a9bb00014da2003f Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 20:04:02 -0500 Subject: [PATCH 08/12] reverted pre-commit file --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4df525f1..e5925cd4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,4 +16,3 @@ repos: rev: v1.6.0 hooks: - id: docformatter - args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99] From b3144c4c025df5f17fa513646198e0fe8d2fb85f Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 20:04:19 -0500 Subject: [PATCH 09/12] reverted pre-commit file --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e5925cd4..4df525f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,3 +16,4 @@ repos: rev: v1.6.0 hooks: - id: docformatter + args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99] From a6f70835f1b36f2edbe6e9f421877bb0bcb06e8e Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 20:04:40 -0500 Subject: [PATCH 10/12] reverted pre-commit file --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4df525f1..23e9ad8a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,6 @@ repos: - id: isort args: ["--profile", "black", "--filter-files"] - - repo: https://github.com/myint/docformatter rev: v1.6.0 hooks: From 836fc892aeb808c80278abcc3a168b5c4726467d Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 20:05:16 -0500 Subject: [PATCH 11/12] reverted pre-commit file --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 23e9ad8a..fcc25da4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,4 +15,4 @@ repos: rev: v1.6.0 hooks: - id: docformatter - args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99] + args: [--in-place, --wrap-summaries=99, --wrap-descriptions=99] \ No newline at end of file From 30d993be25ea9504887ef31aa38f6fca468a844a Mon Sep 17 00:00:00 2001 From: Danny Rosen Date: Wed, 11 Dec 2024 20:26:52 -0500 Subject: [PATCH 12/12] updated pytorch versions --- test/torch_requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/torch_requirements.txt b/test/torch_requirements.txt index 84d87bb6..00faaa7e 100644 --- a/test/torch_requirements.txt +++ b/test/torch_requirements.txt @@ -1,5 +1,5 @@ --index-url https://download.pytorch.org/whl/cpu -torch==2.1.0 -torchaudio==2.1.0 -torchvision==0.16.0 +torch==2.3.0 +torchaudio==2.3.0 +torchvision==0.18.0 torchdata<=0.8.0