Migrate to pyproject.toml and common workflows

.github/workflows/README.md

@@ -1,29 +1,28 @@
 # Continous Integration Workflows
 
-This package implements different workflows for CI.
+This package implements different workflows for CI, based on our organisation's common workflows.
 They are organised as follows.
 
 ### Documentation
 
 The `documentation` workflow triggers on any push to master, builds the documentation and pushes it to the `gh-pages` branch (if the build is successful).
-It runs on `ubuntu-latest` and Python version, `Python 3.9`.
 
 ### Testing Suite
 
 Tests are ensured in the `tests` workflow, which triggers on all pushes.
-It runs on a matrix of all supported operating systems (ubuntu-20.04, ubuntu-22.04, windows-latest and macos-latest) for all supported Python versions (currently `3.8`, `3.9` and `3.10`).
+It runs on a matrix of all supported operating systems for all supported Python versions.
 
 ### Test Coverage
 
 Test coverage is calculated in the `coverage` wokflow, which triggers on pushes to `master` and any push to a `pull request`.
-It runs on `ubuntu-latest` & the Python version (`Python 3.9`), and reports the coverage results of the test suite to `CodeClimate`.
+It reports the coverage results of the test suite to `CodeClimate`.
 
 ### Regular Testing
 
 A `cron` workflow triggers every Monday at 3am (UTC time) and runs the full testing suite, on all available operating systems and supported Python versions.
-It also runs on `Python 3.x` so that newly released Python versions that would break tests are automatically detected.
+It also runs on `Python 3.x` so that newly released Python versions that would break tests are automatically included.
 
 ### Publishing
 
-Publishing to `PyPI` is done through the `publish` workflow, which triggers anytime a `release` is made of the Github repository.
+Publishing to `PyPI` is done through the `publish` workflow, which triggers anytime a `release` is made of the GitHub repository.
 It builds a `wheel`, checks it, and pushes to `PyPI` if checks are successful.

.github/workflows/coverage.yml

@@ -1,10 +1,6 @@
 # Runs all tests and pushes coverage report to codeclimate
 name: Coverage
 
-defaults:
-  run:
-    shell: bash
-
 on:  # Runs on all push events to master branch and any push related to a pull request
   push:
     branches:
@@ -13,60 +9,8 @@ on:  # Runs on all push events to master branch and any push related to a pull r
 
 jobs:
   coverage:
-    name: ${{ matrix.os }} / ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.9]
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/setup.py'
-
-      - name: Get full Python version
-        id: full-python-version
-        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
-
-      - name: Upgrade pip, setuptools and wheel
-        run: python -m pip install --upgrade pip setuptools wheel
-
-      - name: Install package
-        run: python -m pip install '.[test]'
-
-      - name: Set up env for CodeClimate (push)
-        run: |
-          echo "GIT_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
-          echo "GIT_COMMIT_SHA=$GITHUB_SHA" >> $GITHUB_ENV
-        if: github.event_name == 'push'
-
-      - name: Set up env for CodeClimate (pull_request)
-        env:
-          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: |
-          echo "GIT_BRANCH=$GITHUB_HEAD_REF" >> $GITHUB_ENV
-          echo "GIT_COMMIT_SHA=$PR_HEAD_SHA" >> $GITHUB_ENV
-        if: github.event_name == 'pull_request'
-
-      - name: Prepare CodeClimate binary
-        env:
-          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-        run: |
-          curl -LSs 'https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64' >./cc-test-reporter;
-          chmod +x ./cc-test-reporter
-          ./cc-test-reporter before-build
-
-      - name: Run tests
-        run: python -m pytest -m "not cern_network" --mpl --cov-report xml --cov=pylhc
-
-      - name: Push Coverage to CodeClimate
-        if: ${{ success() }}  # only if tests were successful
-        env:
-          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-        run: ./cc-test-reporter after-build
+      uses: pylhc/.github/.github/workflows/coverage.yml@master
+      with:
+        src-dir: pylhc
+        pytest-options: -m "not cern_network" --mpl
+      secrets: inherit

.github/workflows/cron.yml

@@ -1,43 +1,13 @@
-# Runs all tests on master monday at 3 am (UTC time)
+# Runs all tests on master on Mondays at 3 am (UTC time)
 name: Cron Testing
 
-defaults:
-  run:
-    shell: bash
 
-on:  # Runs on master branch on Mondays at 3am UTC time
+on: 
   schedule:
     - cron:  '* 3 * * mon'
 
 jobs:
-  tests:
-    name: ${{ matrix.os }} / ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest]
-        # Make sure to escape 3.10 with quotes so it doesn't get interpreted as float 3.1 by GA's parser
-        python-version: [3.8, 3.9, "3.10", "3.11", 3.x]  # crons should always run latest python hence 3.x
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/setup.py'
-
-      - name: Get full Python version
-        id: full-python-version
-        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
-
-      - name: Upgrade pip, setuptools and wheel
-        run: python -m pip install --upgrade pip setuptools wheel
-
-      - name: Install package
-        run: python -m pip install '.[test]'
-
-      - name: Run tests
-        run: python -m pytest -m "not cern_network" --mpl --cov-report xml --cov=pylhc
+    tests:
+      uses: pylhc/.github/.github/workflows/cron.yml@master
+      with:
+        pytest-options: -m "not cern_network" --mpl

.github/workflows/documentation.yml

@@ -3,10 +3,6 @@
 # The build is published to github pages if the triggering event is a push to the master branch (PR merge)
 name: Build and upload documentation
 
-defaults:
-  run:
-    shell: bash
-
 on:  # Runs on any push event in a PR or any push event to master
   pull_request:
   push:
@@ -15,46 +11,4 @@ on:  # Runs on any push event in a PR or any push event to master
 
 jobs:
   documentation:
-    name: ${{ matrix.os }} / ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.9]
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/setup.py'
-
-      - name: Get full Python version
-        id: full-python-version
-        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
-
-      - name: Upgrade pip, setuptools and wheel
-        run: python -m pip install --upgrade pip setuptools wheel
-
-      - name: Install package
-        run: python -m pip install '.[doc]'
-
-      - name: Build documentation
-        run: python -m sphinx -b html doc ./doc_build -d ./doc_build
-
-      - name: Upload build artifacts  # upload artifacts so reviewers can have a quick look without building documentation from the branch locally
-        uses: actions/upload-artifact@v3
-        if: success() && github.event_name == 'pull_request'  # only for pushes in PR
-        with:
-          name: site-build
-          path: doc_build
-          retention-days: 5
-
-      - name: Upload documentation to gh-pages
-        if: success() && github.ref == 'refs/heads/master'  # only for pushes to master
-        uses: JamesIves/github-pages-deploy-action@v4
-        with:
-          folder: doc_build
+        uses: pylhc/.github/.github/workflows/documentation.yml@master

.github/workflows/publish.yml

@@ -1,50 +1,11 @@
 # Publishes to PyPI upon creation of a release
 name: Upload Package to PyPI
 
-defaults:
-  run:
-    shell: bash
-
 on:  # Runs everytime a release is added to the repository
   release:
     types: [created]
 
 jobs:
   deploy:
-    name: ${{ matrix.os }} / ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.9]
-
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/setup.py'
-
-      - name: Get full Python version
-        id: full-python-version
-        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
-
-      - name: Upgrade pip, setuptools, wheel, build and twine
-        run: python -m pip install --upgrade pip setuptools wheel build twine
-
-      - name: Build and check build
-        run: |
-          python -m build
-          twine check dist/*
-
-      - name: Publish
-        if: ${{ success() }}
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          twine upload dist/*
+    uses: pylhc/.github/.github/workflows/publish.yml@master
+    secrets: inherit

.github/workflows/tests.yml

@@ -1,42 +1,17 @@
-# Runs all tests not flagged as "cern_network" with a pytest marker
-name: Tests
+# Runs all tests 
+name: All Tests
 
 defaults:
   run:
     shell: bash
 
-on:  [push]  # Runs on all push events to any branch
-
+on:  # Runs on any push event to any branch except master (the coverage workflow takes care of that)
+  push:
+    branches-ignore:
+      - 'master'
 
 jobs:
   tests:
-    name: ${{ matrix.os }} / ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest]
-        # Make sure to escape 3.10 with quotes so it doesn't get interpreted as float 3.1 by GA's parser
-        python-version: [3.8, 3.9, "3.10", "3.11"]
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/setup.py'
-
-      - name: Get full Python version
-        id: full-python-version
-        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
-
-      - name: Upgrade pip, setuptools and wheel
-        run: python -m pip install --upgrade pip setuptools wheel
-
-      - name: Install package
-        run: python -m pip install '.[test]'
-
-      - name: Run basic tests
-        run: python -m pytest -m "not cern_network" --mpl
+    uses: pylhc/.github/.github/workflows/tests.yml@master
+    with:
+      pytest-options: -m "not cern_network" --mpl

.gitignore

@@ -228,5 +228,4 @@ Temporary Items
 .docs/gen_modules
 .docs/gen_modules/
 
-doc_build/
-pyproject.toml
+doc_build/

pylhc/bpm_calibration.py

@@ -52,10 +52,7 @@
     default: ``beta``
 """
 from pathlib import Path
-from typing import List, Tuple
 
-import numpy as np
-import pandas as pd
 import tfs
 from generic_parser import EntryPointParameters, entrypoint
 from omc3.optics_measurements.constants import EXT

pylhc/bsrt_analysis.py

@@ -365,7 +365,7 @@ def plot_crosssection_for_timesteps(opt, bsrt_df):
         fig.suptitle(f"Timestamp: {timestamp}")
 
         ax[0].imshow(_reshaped_imageset(data_row), cmap="hot", interpolation="nearest")
-        ax[0].set_title(f"2D Pixel count")
+        ax[0].set_title("2D Pixel count")
 
         ax[1].plot(data_row["projPositionSet1"], data_row["projDataSet1"], color="darkred")
         ax[1].plot(

pylhc/calibration/beta.py

@@ -8,7 +8,7 @@
 
 """
 from pathlib import Path
-from typing import Dict, List, Sequence, Tuple
+from typing import Dict, Sequence, Tuple
 
 import numpy as np
 import pandas as pd
@@ -17,7 +17,7 @@
 from omc3.utils import logging_tools
 from scipy.optimize import curve_fit
 
-from pylhc.constants.calibration import BETA_STAR_ESTIMATION, BPMS, IPS, LABELS, TFS_INDEX
+from pylhc.constants.calibration import BETA_STAR_ESTIMATION, BPMS, LABELS, TFS_INDEX
 from pylhc.constants.general import PLANES
 
 LOG = logging_tools.get_logger(__name__)

pylhc/calibration/dispersion.py

@@ -26,12 +26,9 @@
     BPMS,
     D_BPMS,
     D,
-    IPS,
     LABELS,
-    ND,
     TFS_INDEX,
 )
-from pylhc.constants.general import PLANES
 import tfs
 
 
@@ -185,10 +182,10 @@ def get_calibration_factors_from_dispersion(
         dispersion = dict()
         normalised_dispersion = dict()
 
-        dispersion["amp"] = dispersion_tfs.reindex(bpms)[f"DX"]
+        dispersion["amp"] = dispersion_tfs.reindex(bpms)["DX"]
         dispersion["amp_err"] = dispersion_tfs.reindex(bpms)[f"{ERR}{D}X"]
 
-        dispersion["phase"] = norm_dispersion_tfs.reindex(bpms)[f"DX"]
+        dispersion["phase"] = norm_dispersion_tfs.reindex(bpms)["DX"]
         dispersion["phase_err"] = norm_dispersion_tfs.reindex(bpms)[f"{ERR}{D}X"]
 
         # Compute the calibration factors using the dispersion from phase and amp