From 90cd55ac9273b98cbe86c211e4cf49c19777d855 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Tue, 20 May 2025 22:16:13 -0700 Subject: [PATCH 01/14] relax version constraint for pandas --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0a26558..6dc84f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] dependencies = [ "mip==1.16rc0", - "pandas>=2.2.2", + "pandas>=1.5", "xmltodict==0.12.0", "requests>=2.0.0", ] From 8fbee66b2814138811d32b1bb3b4eb8b0884f15d Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Tue, 20 May 2025 22:16:13 -0700 Subject: [PATCH 02/14] avoid using include_groups --- src/nempy/markets.py | 2 +- .../spot_market_backend/solver_interface.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/nempy/markets.py b/src/nempy/markets.py index be76993..a27eb38 100644 --- a/src/nempy/markets.py +++ b/src/nempy/markets.py @@ -3167,7 +3167,7 @@ def not_closest_three(df): df = df.iloc[3:, :] return df - vars_to_remove = vars.groupby(['interconnector', 'link'], as_index=False).apply(not_closest_three, include_groups=False) + vars_to_remove = vars.groupby(['interconnector', 'link'], as_index=False)[['distance', 'variable_id']].apply(not_closest_three) si.disable_variables(vars_to_remove.loc[:, ['variable_id']]) def get_constraint_set_names(self): diff --git a/src/nempy/spot_market_backend/solver_interface.py b/src/nempy/spot_market_backend/solver_interface.py index 091c2d4..cd1a01a 100644 --- a/src/nempy/spot_market_backend/solver_interface.py +++ b/src/nempy/spot_market_backend/solver_interface.py @@ -114,26 +114,24 @@ def add_sos_type_2(self, sos_variables, sos_id_columns, position_column): """ - # Function that adds sets to mip model. - def add_sos_vars(sos_group): - self.mip_model.add_sos(list(zip(sos_group['vars'], sos_group[position_column])), 2) - # For each variable_id get the variable object from the mip model sos_variables['vars'] = sos_variables['variable_id'].apply(lambda x: self.variables[x]) + # Break up the sets based on their id and add them to the model separately. - sos_variables.groupby(sos_id_columns).apply(add_sos_vars, include_groups=False) + for _, sos_group in sos_variables.groupby(sos_id_columns): + self.mip_model.add_sos(list(zip(sos_group['vars'], sos_group[position_column])), 2) + # This is a hack to make sure mip knows there are binary constraints. self.mip_model.add_var(var_type=BINARY, obj=0.0) def add_sos_type_1(self, sos_variables): - # Function that adds sets to mip model. - def add_sos_vars(sos_group): - self.mip_model.add_sos(list(zip(sos_group['vars'], [1.0 for i in range(len(sos_variables['vars']))])), 1) - # For each variable_id get the variable object from the mip model sos_variables['vars'] = sos_variables['variable_id'].apply(lambda x: self.variables[x]) + # Break up the sets based on their id and add them to the model separately. - sos_variables.groupby('sos_id').apply(add_sos_vars) + for _, sos_group in sos_variables.groupby('sos_id'): + self.mip_model.add_sos(list(zip(sos_group['vars'], [1.0 for i in range(len(sos_variables['vars']))])), 1) + # This is a hack to make mip knows there are binary constraints. self.mip_model.add_var(var_type=BINARY, obj=0.0) From b2e751e1b1768348334b068bf3f8d292e4af3c69 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Tue, 20 May 2025 22:16:13 -0700 Subject: [PATCH 03/14] extend tests to cover get_region_dispatch_summary --- tests/test_markets.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_markets.py b/tests/test_markets.py index 086ce93..f29c358 100644 --- a/tests/test_markets.py +++ b/tests/test_markets.py @@ -55,8 +55,14 @@ def test_one_region_energy_market(): 'dispatch': [40.0, 20.0] }) + expected_region_dispatch_summary = pd.DataFrame({ + 'region': ['NSW'], + 'dispatch': [60.0], + }) + assert_frame_equal(market.get_energy_prices(), expected_prices) assert_frame_equal(market.get_unit_dispatch(), expected_dispatch) + assert_frame_equal(market.get_region_dispatch_summary(), expected_region_dispatch_summary) def test_two_region_energy_market(): @@ -111,8 +117,14 @@ def test_two_region_energy_market(): 'dispatch': [60.0, 80.0] }) + expected_region_dispatch_summary = pd.DataFrame({ + 'region': ['NSW', 'VIC'], + 'dispatch': [60.0, 80.0], + }) + assert_frame_equal(market.get_energy_prices(), expected_prices) assert_frame_equal(market.get_unit_dispatch(), expected_dispatch) + assert_frame_equal(market.get_region_dispatch_summary(), expected_region_dispatch_summary) def test_one_interconnector(): @@ -204,9 +216,18 @@ def constant_losses(flow): 'losses': [(90.0/0.975) * 0.05] }) + expected_region_dispatch_summary = pd.DataFrame({ + 'region': ['NSW', 'VIC'], + 'dispatch': [94.615385, 0.0], + 'inflow': [-92.307692, 92.307692], + 'transmission_losses': [0.0, 0.0], + 'interconnector_losses': [2.307692, 2.307692], + }) + assert_frame_equal(market.get_energy_prices(), expected_prices) assert_frame_equal(market.get_unit_dispatch(), expected_dispatch) assert_frame_equal(market.get_interconnector_flows(), expected_interconnector_flow) + assert_frame_equal(market.get_region_dispatch_summary(), expected_region_dispatch_summary) def test_one_region_energy_and_raise_regulation_markets(): @@ -1049,4 +1070,4 @@ def test_setting_constraint_on_unit_with_no_bid_volume_doesnt_raise_error(): }) assert_frame_equal(market.get_energy_prices(), expected_prices) - assert_frame_equal(market.get_unit_dispatch(), expected_dispatch) \ No newline at end of file + assert_frame_equal(market.get_unit_dispatch(), expected_dispatch) From 1fdd62a87511ceaf3df19cb149518aa8d0f476d1 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 12:49:59 -0700 Subject: [PATCH 04/14] test with older pandas and numpy --- .github/workflows/cicd.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 7b577df..d55172a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -19,6 +19,10 @@ jobs: matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + version-constraints: + - "" + - "'pandas==1.*' 'numpy==1.*'" + # Necessary for poetry & Windows defaults: run: @@ -36,6 +40,11 @@ jobs: - name: Sync dependencies run: uv sync + - name: Resync with restricted versions + if: ${{ matrix.version-constraints != '' }} + run: | + uv pip install ${{ matrix.version-constraints }} --reinstall --strict + - name: Run tests run: uv run --frozen pytest tests src -ra --doctest-modules From 60d0c8725bda071baf16f73a75adb95c4e484116 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 12:54:13 -0700 Subject: [PATCH 05/14] fix workflow file --- .github/workflows/cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d55172a..d1133eb 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -19,9 +19,9 @@ jobs: matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - version-constraints: - - "" - - "'pandas==1.*' 'numpy==1.*'" + version-constraints: + - "" + - "'pandas==1.*' 'numpy==1.*'" # Necessary for poetry & Windows defaults: From c0eeda1cf492ed55804c2a258154f43a7df66758 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 13:09:16 -0700 Subject: [PATCH 06/14] turn off fail fast for a moment --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d1133eb..a8fe477 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -15,7 +15,7 @@ jobs: # Matrix testing to test across OSs and Python versions # Fail-fast: fail the entire job as soon as anything fails strategy: - fail-fast: true + fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From ca2310d103442649d5d4113a6916c331effd7086 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 15:59:52 -0700 Subject: [PATCH 07/14] upgrade action versions --- .github/workflows/cicd.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index a8fe477..94e4d5c 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -30,10 +30,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@v6 with: enable-cache: true @@ -61,10 +61,10 @@ jobs: steps: # Checkout repo - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Install uv with cache - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@v6 with: enable-cache: true # Sync dependencies From efc9c53450e86a634c8c8973d077e04861d1f076 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 16:15:38 -0700 Subject: [PATCH 08/14] try to get macos builds to pickup the correct python --- .github/workflows/cicd.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 94e4d5c..54ec363 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -32,6 +32,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install the latest version of uv uses: astral-sh/setup-uv@v6 with: From f67a920d751b6f3d23acd798920ad3bc57b17fb1 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 16:26:08 -0700 Subject: [PATCH 09/14] drop 3.8 add 3.13, reenable fail fast --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 54ec363..a4ab796 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -15,10 +15,10 @@ jobs: # Matrix testing to test across OSs and Python versions # Fail-fast: fail the entire job as soon as anything fails strategy: - fail-fast: false + fail-fast: true matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] version-constraints: - "" - "'pandas==1.*' 'numpy==1.*'" From 96c96ae89c72c3cc1f5718e9e916f698c3c32d7c Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 16:42:42 -0700 Subject: [PATCH 10/14] add exclude --- .github/workflows/cicd.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index a4ab796..c33c872 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -15,13 +15,17 @@ jobs: # Matrix testing to test across OSs and Python versions # Fail-fast: fail the entire job as soon as anything fails strategy: - fail-fast: true + fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] version-constraints: - "" - "'pandas==1.*' 'numpy==1.*'" + exclude: + - os: macos-latest + python-version: '3.13' + version-constraints: "'pandas==1.*' 'numpy==1.*'" # Necessary for poetry & Windows defaults: From 5368783008932ec8d30308a2ef9eb8ecd8bc3560 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 16:43:53 -0700 Subject: [PATCH 11/14] fix yaml --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index c33c872..9dadeb2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -22,7 +22,7 @@ jobs: version-constraints: - "" - "'pandas==1.*' 'numpy==1.*'" - exclude: + exclude: - os: macos-latest python-version: '3.13' version-constraints: "'pandas==1.*' 'numpy==1.*'" From 6f597d0437b03c3aca05c1fd078dbef1cfa23e4a Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Wed, 21 May 2025 16:59:58 -0700 Subject: [PATCH 12/14] add some excludes for 3.13 --- .github/workflows/cicd.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 9dadeb2..b8419d0 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -15,7 +15,7 @@ jobs: # Matrix testing to test across OSs and Python versions # Fail-fast: fail the entire job as soon as anything fails strategy: - fail-fast: false + fail-fast: true matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] @@ -23,11 +23,14 @@ jobs: - "" - "'pandas==1.*' 'numpy==1.*'" exclude: - - os: macos-latest - python-version: '3.13' + # pandas 1.x is not compatible python 3.13 + - python-version: "3.13" version-constraints: "'pandas==1.*' 'numpy==1.*'" + # uv run segfaults for windows on python 3.13 + - os: "windows-latest" + python-version: '3.13' - # Necessary for poetry & Windows + # Necessary for Windows defaults: run: shell: bash From 81d26c1aaf2e92f5b306a9bbddedc61cc8f30ff1 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Thu, 16 Oct 2025 15:54:07 -0700 Subject: [PATCH 13/14] improve error message (forgot pass all entries to format) --- .../dataframe_validator.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/nempy/spot_market_backend/dataframe_validator.py b/src/nempy/spot_market_backend/dataframe_validator.py index 29c3e71..4ee7b18 100644 --- a/src/nempy/spot_market_backend/dataframe_validator.py +++ b/src/nempy/spot_market_backend/dataframe_validator.py @@ -71,12 +71,25 @@ def _check_data_type(self, series): if not all(series.apply(lambda x: callable(x))): raise ColumnDataTypeError('All elements of column {} should have type callable'.format(self.name)) elif self.data_type != series.dtype: - raise ColumnDataTypeError('Column {} should have type {}'.format(self.name, self.data_type)) + raise ColumnDataTypeError( + 'Column {} should have type {}, but found {}'.format( + self.name, + self.data_type, + series.dtype, + ) + ) def _check_allowed_values(self, series): if self.allowed_values is not None: - if not series.isin(self.allowed_values).all(): - raise ColumnValues("The column {} can only contain the values {}.".format(self.name, self.allowed_values)) + invalid_values = ~series.isin(self.allowed_values) + if invalid_values.any(): + raise ColumnValues( + "The column {} has invalid entries ({}). Must only contain the values {}.".format( + self.name, + sorted(series[invalid_values].unique().tolist()), + self.allowed_values, + ) + ) def _check_is_real_number(self, series): if self.must_be_real_number: From c006b96316f0bca52bcfcdcd721a3c43fd1de940 Mon Sep 17 00:00:00 2001 From: Gabriel Hackebeil Date: Thu, 16 Oct 2025 21:57:37 -0700 Subject: [PATCH 14/14] skip macos test build for python 3.13 --- .github/workflows/cicd.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index b8419d0..4b2cac3 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -9,7 +9,7 @@ on: release: types: [ created ] workflow_dispatch: # manual trigger - + jobs: test: # Matrix testing to test across OSs and Python versions @@ -29,6 +29,10 @@ jobs: # uv run segfaults for windows on python 3.13 - os: "windows-latest" python-version: '3.13' + # numpy build issues on macos-latest + - os: "macos-latest" + python-version: '3.13' + # Necessary for Windows defaults: @@ -59,7 +63,7 @@ jobs: - name: Run tests run: uv run --frozen pytest tests src -ra --doctest-modules - + publish: if: github.event_name == 'release' && github.event.action == 'created' needs: test