restructure Readme content so we have first things first

.github/workflows/lint-and-test.yml

@@ -6,35 +6,37 @@ on: push
 jobs:
   check:
     runs-on: ubuntu-latest
-    name: Check (on Python3.9)
+    name: Check (on Python 3.11)
     steps:
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.9
-      - uses: actions/checkout@v2
-      - uses: pre-commit/action@v2.0.0
+          python-version: 3.11
+      - uses: actions/checkout@v3
+      - uses: pre-commit/action@v3.0.0
+
 
   test:
     needs: check
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        py_version: [ '3.9' ]
+        py_version: [ '3.11' ]
     name: "Test (on Python ${{ matrix.py_version }})"
     steps:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.py_version }}
       - name: Check out src from Git
-        uses: actions/checkout@v2
-      - name: Get history and tags for SCM versioning to work
+        uses: actions/checkout@v3
+      - name: Install SQL extensions
         run: |
-          git fetch --prune --unshallow
-          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
           sudo apt-get update
           sudo apt-get -y install postgresql-client
           psql -h $PGHOST -p $PGPORT --file scripts/load-psql-extensions.sql -U $PGUSER $PGDB;
+      - name: Install necessary items for netcdf
+        run: |
+          sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev
       - run: make test
     env:
       PGHOST: 127.0.0.1
@@ -47,7 +49,7 @@ jobs:
       # Label used to access the service container
       postgres:
         # Docker Hub image
-        image: postgres:12.5 
+        image: postgres:14.17 
         env:
           POSTGRES_USER: flexmeasures_test
           POSTGRES_PASSWORD: flexmeasures_test

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
 -   repo: https://github.com/pycqa/flake8
-    rev: 6.0.0  # New version tags can be found here: https://github.com/pycqa/flake8/tags
+    rev: 7.1.1  # New version tags can be found here: https://github.com/pycqa/flake8/tags
     hooks:
     - id: flake8
       name: flake8 (code linting)
 -   repo: https://github.com/psf/black
-    rev: 22.10.0  # New version tags can be found here: https://github.com/psf/black/tags
+    rev: 24.8.0  # New version tags can be found here: https://github.com/psf/black/tags
     hooks:
     - id: black
       name: black (code formatting)

Makefile

@@ -25,7 +25,7 @@ install-deps:
 	pip-sync requirements/app.txt
 
 install-flexmeasures-weather:
-	python setup.py develop
+	pip install -e .
 
 install-pip-tools:
 	pip3 install -q "pip-tools>=6.2"

README.md

@@ -1,9 +1,62 @@
-# FLEXMEASURES-WEATHER - a plugin for FlexMeasures to integrate multiple Weather API services
+# FLEXMEASURES-WEATHER - a plugin for FlexMeasures to integrate weather forecasts
 
-### Configuration
 
 This plugin currently supports two Weather API services: [OpenWeatherMap One Call API](https://openweathermap.org/api/one-call-3) and [Weather API](https://www.weatherapi.com/). The configuration is controlled via your FlexMeasures config file.
 
+
+## Usage
+
+To register a new weather sensor:
+
+`flexmeasures weather register-weather-sensor --name "wind speed" --latitude 30 --longitude 40`
+
+Currently supported: wind speed, temperature & irradiance.
+
+To collect weather forecasts:
+
+`flexmeasures weather get-weather-forecasts --location 30,40`
+
+This saves forecasts for your registered sensors in the database.
+
+Use the `--help`` option for more options, e.g. for specifying two locations and requesting that a number of weather stations cover the bounding box between them (where the locations represent top left and bottom right).
+
+Notes about weather sensor setup: 
+
+- Weather sensors are public assets in FlexMeasures. They are accessible by all accounts on a FlexMeasures server.
+- The resolution is one hour. Weather also supports minutely data within the upcoming hour(s), but that is not supported here.
+
+An alternative usage is to save raw results in JSON files (for later processing), like this:
+
+`flexmeasures weather get-weather-forecasts --location 30,40 --store-as-json-files --region somewhere`
+
+This saves the complete response from the Weather Provider in a local folder (i.e. no sensor registration needed, this is a direct way to use Weather APIs, without FlexMeasures integration). `region` will become a subfolder.
+
+Finally, note that these APIs allow free calls, but not without limits.
+For instance, currently 1000 free calls per day can be made to the OpenWeatherMap API,
+so you can make a call every 15 minutes for up to 10 locations or every hour for up to 40 locations (or get a paid account).
+
+
+## Setup
+
+### Installation
+
+To add as plugin to an existing FlexMeasures system, add "/path/to/flexmeasures-weather-repo/flexmeasures_weather" to your FlexMeasures config file,
+using the FLEXMEASURES_PLUGINS setting (a list).
+
+Alternatively, if you installed this plugin as a package (e.g. via `python setup.py install`, `pip install -e` or `pip install flexmeasures_weather` after this project is on Pypi), then "flexmeasures_weather" suffices.
+
+To enable weather forecast functionality, two PostgreSQL extensions must be installed. Run the following SQL commands in your database:
+
+```
+CREATE EXTENSION IF NOT EXISTS cube;
+CREATE EXTENSION IF NOT EXISTS earthdistance;
+```
+
+These extensions provide support for geographical calculations such as `ll_to_earth` and `earth_distance`, which we use to find the nearest weather station asset.
+
+
+### Configuration
+
 Add the following entries to your config:
 
 ```ini
@@ -57,68 +110,13 @@ To expand the plugin's coverage to additional weather API services:
 > This modular structure allows for seamless integration of additional services while maintaining consistency and clarity in data handling.
 
 
-## Usage
-
-To register a new weather sensor:
-
-`flexmeasures weather register-weather-sensor --name "wind speed" --latitude 30 --longitude 40`
-
-Currently supported: wind speed, temperature & irradiance.
-
-Notes about weather sensor setup: 
-
-- Weather sensors are public. They are accessible by all accounts on a FlexMeasures server. TODO: maybe limit this to a list of account roles.
-- The resolution is one hour. Weather also supports minutely data within the upcoming hour(s), but that is not supported here.
-
-To collect weather forecasts:
-
-Enable required Postgres Extensions
-
-To enable weather forecast functionality, two PostgreSQL extensions must be installed. Run the following SQL commands in your database:
-
-```
-CREATE EXTENSION IF NOT EXISTS cube;
-CREATE EXTENSION IF NOT EXISTS earthdistance;
-```
-
-These extensions provide support for geographical calculations such as ll_to_earth and earth_distance, which are used to determine proximity between coordinates.
-
-`flexmeasures weather get-weather-forecasts --location 30,40`
-
-This saves forecasts for your registered sensors in the database.
-
-Use the `--help`` option for more options, e.g. for specifying two locations and requesting that a number of weather stations cover the bounding box between them (where the locations represent top left and bottom right).
-
-An alternative usage is to save raw results in JSON files (for later processing), like this:
-
-`flexmeasures weather get-weather-forecasts --location 30,40 --store-as-json-files --region somewhere`
-
-This saves the complete response from the Weather Provider in a local folder (i.e. no sensor registration needed, this is a direct way to use Weather, without FlexMeasures integration). `region` will become a subfolder.
-
-Finally, note that currently 1000 free calls per day can be made to the OpenWeatherMap API,
-so you can make a call every 15 minutes for up to 10 locations or every hour for up to 40 locations (or get a paid account).
-
-
-## Installation
-
-To install locally, run
-
-    make install
-
-To add as plugin to an existing FlexMeasures system, add "/path/to/FLEXMEASURES-WEATHER/flexmeasures_weather" to your FlexMeasures (>v0.7.0dev8) config file,
-using the FLEXMEASURES_PLUGINS setting (a list).
-
-Alternatively, if you installed this plugin as a package (e.g. via `python setup.py install`, `pip install -e` or `pip install flexmeasures_weather` after this project is on Pypi), then "flexmeasures_weather" suffices.
-
-
-
 ## Development
 
 We use pre-commit to keep code quality up.
 
 Install necessary tools with:
 
-    pip install pre-commit black flake8 mypy
+    pip install pre-commit
     pre-commit install
 
 or:

flexmeasures_weather/__init__.py

@@ -8,7 +8,7 @@
 """
 
 
-from importlib_metadata import version, PackageNotFoundError
+from importlib.metadata import version, PackageNotFoundError
 
 from flask import Blueprint
 

flexmeasures_weather/cli/commands.py

@@ -164,9 +164,7 @@ def collect_weather_data(location, asset_id, store_in_db, num_cells, method, reg
 
     api_key = str(current_app.config.get("WEATHERAPI_KEY", ""))
     if api_key == "":
-        raise Exception(
-            "[FLEXMEASURES-WEATHER] Setting WEATHERAPI_KEY not available."
-        )
+        raise Exception("[FLEXMEASURES-WEATHER] Setting WEATHERAPI_KEY not available.")
     if asset_id is not None:
         locations = [get_location_by_asset_id(asset_id)]
     elif location is not None:

flexmeasures_weather/cli/tests/test_get_forecasts.py

@@ -63,7 +63,5 @@ def test_get_weather_forecasts_no_close_sensors(
             ["--location", f"{weather_station.latitude-5},{weather_station.longitude}"],
         )
         print(result.output)
-        assert (
-            "Reported task get-weather-forecasts status as True" in result.output
-        )
+        assert "Reported task get-weather-forecasts status as True" in result.output
         assert "no sufficiently close weather sensor found" in caplog.text

flexmeasures_weather/utils/locating.py

@@ -123,6 +123,7 @@ def get_location_by_asset_id(asset_id: int) -> Tuple[float, float]:
         )
     if asset is None:
         raise Exception(
-            "[FLEXMEASURES-WEATHER] No asset found for the given asset id %s." % asset_id
+            "[FLEXMEASURES-WEATHER] No asset found for the given asset id %s."
+            % asset_id
         )
     return (asset.latitude, asset.longitude)