From 17795d5b629181a60c7731dfc50e7242edae7ca5 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 15 Oct 2024 13:36:15 -0500 Subject: [PATCH 01/10] switch json schema to basic github actions python install --- .github/workflows/check_json.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check_json.yml b/.github/workflows/check_json.yml index 60d61175..f5f45fd5 100644 --- a/.github/workflows/check_json.yml +++ b/.github/workflows/check_json.yml @@ -8,13 +8,12 @@ jobs: shell: bash -el {0} steps: - name: check out repository code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: set up conda environment - uses: conda-incubator/setup-miniconda@v2 + uses: actions/setup-python@v5 with: - auto-update-conda: true - auto-activate-base: false - activate-environment: checkjson-env + python-version: '3.12' + cache: 'pip' # caching pip dependencies - name: Install check-jsonschema run: | pip install check-jsonschema From 0a3b23f898b49b52dd7a9c12e66bc9ed5a3e0a34 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 15 Oct 2024 13:39:20 -0500 Subject: [PATCH 02/10] update check_formatting job --- .github/workflows/check_formatting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml index b6f8ec27..c78e86b6 100644 --- a/.github/workflows/check_formatting.yml +++ b/.github/workflows/check_formatting.yml @@ -4,9 +4,9 @@ jobs: formatting_job: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: miniforge-version: latest miniforge-variant: mambaforge From bfe5e7d548df7e8cabd1fda663c12084d825a2fe Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 15:48:23 -0500 Subject: [PATCH 03/10] update black formatting check --- .github/workflows/check_formatting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml index c78e86b6..165ea111 100644 --- a/.github/workflows/check_formatting.yml +++ b/.github/workflows/check_formatting.yml @@ -1,4 +1,4 @@ -name: check_formatting +name: Check Python File Formatting with Black on: [push, pull_request] jobs: formatting_job: @@ -9,7 +9,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: miniforge-version: latest - miniforge-variant: mambaforge + miniforge-variant: miniforge channel-priority: strict channels: conda-forge show-channel-urls: true From 266971f376977aa26c25173354adc46a8392cdd6 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 15:51:33 -0500 Subject: [PATCH 04/10] update black formatting check --- .github/workflows/check_formatting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml index 165ea111..58b8dc5d 100644 --- a/.github/workflows/check_formatting.yml +++ b/.github/workflows/check_formatting.yml @@ -8,12 +8,12 @@ jobs: - name: Set up conda uses: conda-incubator/setup-miniconda@v3 with: - miniforge-version: latest - miniforge-variant: miniforge + activate-environment: tobac_black_formatting channel-priority: strict channels: conda-forge show-channel-urls: true use-only-tar-bz2: true + python-version: 3.12 - name: Install dependencies and check formatting shell: bash -l {0} From c7042462dcdcd75830b65777054f067b3010966a Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 15:52:40 -0500 Subject: [PATCH 05/10] remove python version requirement --- .github/workflows/check_formatting.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml index 58b8dc5d..747d9504 100644 --- a/.github/workflows/check_formatting.yml +++ b/.github/workflows/check_formatting.yml @@ -13,7 +13,6 @@ jobs: channels: conda-forge show-channel-urls: true use-only-tar-bz2: true - python-version: 3.12 - name: Install dependencies and check formatting shell: bash -l {0} From a88746152a5f4fce87423da88e9b63a5fc8d2804 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 15:55:06 -0500 Subject: [PATCH 06/10] switch to micromamba --- .github/workflows/check_formatting.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml index 747d9504..527c5a0b 100644 --- a/.github/workflows/check_formatting.yml +++ b/.github/workflows/check_formatting.yml @@ -6,16 +6,10 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up conda - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: tobac_black_formatting - channel-priority: strict - channels: conda-forge - show-channel-urls: true - use-only-tar-bz2: true + uses: mamba-org/setup-micromamba@v1 - name: Install dependencies and check formatting - shell: bash -l {0} + shell: bash -el {0} run: mamba install --quiet --yes --file requirements.txt black && black --version && From 478f9ffc69ddc14fe6eaa19ef4c0813efd823ccb Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 15:58:02 -0500 Subject: [PATCH 07/10] updates to micromamba-shell --- .github/workflows/check_formatting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml index 527c5a0b..fe5f2255 100644 --- a/.github/workflows/check_formatting.yml +++ b/.github/workflows/check_formatting.yml @@ -4,12 +4,12 @@ jobs: formatting_job: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up conda uses: mamba-org/setup-micromamba@v1 - name: Install dependencies and check formatting - shell: bash -el {0} + shell: micromamba-shell {0} run: mamba install --quiet --yes --file requirements.txt black && black --version && From b006dd7e149f4a763fb3e63374da0f52ab7e63e4 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 16:01:48 -0500 Subject: [PATCH 08/10] switch to using an environment file rather than mamba install --- .github/workflows/check_formatting.yml | 9 ++++++--- environment-ci.yml | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check_formatting.yml b/.github/workflows/check_formatting.yml index fe5f2255..bca4a182 100644 --- a/.github/workflows/check_formatting.yml +++ b/.github/workflows/check_formatting.yml @@ -7,10 +7,13 @@ jobs: - uses: actions/checkout@v4 - name: Set up conda uses: mamba-org/setup-micromamba@v1 - - - name: Install dependencies and check formatting + with: + environment-file: environment-ci.yml + generate-run-shell: true + cache-environment: true + cache-downloads: true + - name: Check formatting shell: micromamba-shell {0} run: - mamba install --quiet --yes --file requirements.txt black && black --version && black tobac --check --diff diff --git a/environment-ci.yml b/environment-ci.yml index 4f0b954b..e0d2bebc 100644 --- a/environment-ci.yml +++ b/environment-ci.yml @@ -14,3 +14,4 @@ dependencies: - trackpy - pytest - typing_extensions + - black From 321ea16e9b6c07a7e8baee7d7cc3698d8f93bbfe Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 16:10:19 -0500 Subject: [PATCH 09/10] reformatted files with black 24.x --- environment-ci.yml | 2 +- tobac/feature_detection.py | 1 - tobac/tests/test_utils_bulk_statistics.py | 44 +++++++++++------------ tobac/utils/bulk_statistics.py | 10 ++++-- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/environment-ci.yml b/environment-ci.yml index e0d2bebc..964e7155 100644 --- a/environment-ci.yml +++ b/environment-ci.yml @@ -1,4 +1,4 @@ -name: pyart-dev +name: tobac-dev channels: - conda-forge dependencies: diff --git a/tobac/feature_detection.py b/tobac/feature_detection.py index 075b8513..0889e6b7 100644 --- a/tobac/feature_detection.py +++ b/tobac/feature_detection.py @@ -1015,7 +1015,6 @@ def feature_detection_multithreshold_timestep( raise ValueError( "Please provide the input parameter statistic to determine what statistics to calculate." ) - track_data = gaussian_filter( track_data, sigma=sigma_threshold diff --git a/tobac/tests/test_utils_bulk_statistics.py b/tobac/tests/test_utils_bulk_statistics.py index 2026f9c4..d50bf539 100644 --- a/tobac/tests/test_utils_bulk_statistics.py +++ b/tobac/tests/test_utils_bulk_statistics.py @@ -153,51 +153,51 @@ def test_bulk_statistics_missing_segments(): ### Test 2D data with time dimension test_data = tb_test.make_simple_sample_data_2D().core_data() common_dset_opts = { - "in_arr": test_data, - "data_type": "iris",} - + "in_arr": test_data, + "data_type": "iris", + } + test_data_iris = tb_test.make_dataset_from_arr( - time_dim_num=0, y_dim_num=1, x_dim_num=2, **common_dset_opts) + time_dim_num=0, y_dim_num=1, x_dim_num=2, **common_dset_opts + ) # detect features threshold = 7 # test_data_iris = testing.make_dataset_from_arr(test_data, data_type="iris") fd_output = tobac.feature_detection.feature_detection_multithreshold( - test_data_iris, - dxy=1000, - threshold=[threshold], - n_min_threshold=100, - target="maximum",) + test_data_iris, + dxy=1000, + threshold=[threshold], + n_min_threshold=100, + target="maximum", + ) # perform segmentation with bulk statistics stats = { - "segment_max": np.max, - "segment_min": min, - "percentiles": (np.percentile, {"q": 95}),} + "segment_max": np.max, + "segment_min": min, + "percentiles": (np.percentile, {"q": 95}), + } out_seg_mask, out_df = tobac.segmentation.segmentation_2D( - fd_output, test_data_iris, dxy=1000, threshold=threshold) + fd_output, test_data_iris, dxy=1000, threshold=threshold + ) - # specify some timesteps we set to zero + # specify some timesteps we set to zero timesteps_to_zero = [1, 3, 10] # 0-based indexing - modified_data = out_seg_mask.data.copy() + modified_data = out_seg_mask.data.copy() # Set values to zero for the specified timesteps for timestep in timesteps_to_zero: modified_data[timestep, :, :] = 0 # Set all values for this timestep to zero # assure that bulk statistics in postprocessing give same result out_segmentation = tb_utils.get_statistics_from_mask( - out_df, out_seg_mask, test_data_iris, statistic=stats) + out_df, out_seg_mask, test_data_iris, statistic=stats + ) assert out_df.time.unique().size == out_segmentation.time.unique().size - - - - - - def test_bulk_statistics_multiple_fields(): """ Test that multiple field input to bulk_statistics works as intended diff --git a/tobac/utils/bulk_statistics.py b/tobac/utils/bulk_statistics.py index f9266dd8..daa430eb 100644 --- a/tobac/utils/bulk_statistics.py +++ b/tobac/utils/bulk_statistics.py @@ -301,9 +301,15 @@ def get_statistics_from_mask( for tt in pd.to_datetime(segmentation_mask.time): # select specific timestep - segmentation_mask_t = segmentation_mask.sel(time=tt, method = 'nearest').data + segmentation_mask_t = segmentation_mask.sel(time=tt, method="nearest").data fields_t = ( - field.sel(time=tt, method = 'nearest', tolerance = np.timedelta64(1000, 'us')).values if "time" in field.coords else field.values + ( + field.sel( + time=tt, method="nearest", tolerance=np.timedelta64(1000, "us") + ).values + if "time" in field.coords + else field.values + ) for field in fields ) From 2ec16420646acde5719a7349ede83b31409a778a Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 17 Oct 2024 16:33:22 -0500 Subject: [PATCH 10/10] merged latest changes and reformatted --- tobac/feature_detection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tobac/feature_detection.py b/tobac/feature_detection.py index e5b95279..72955bb2 100644 --- a/tobac/feature_detection.py +++ b/tobac/feature_detection.py @@ -1135,14 +1135,14 @@ def feature_detection_multithreshold_timestep( labels.ravel()[regions_old[key]] = key # apply function to get statistics based on labeled regions and functions provided by the user # the feature dataframe is updated by appending a column for each metric - + # select which data to use according to statistics_unsmoothed option stats_data = data_i.core_data() if statistics_unsmoothed else track_data - + features_thresholds = get_statistics( features_thresholds, labels, - stats_data, + stats_data, statistic=statistic, index=np.unique(labels[labels > 0]), id_column="idx",