Beamprocess manual override (#106)

* Added BeamProcess Override
* fixed some deprecated use of get_loc, append
* documentation
* github workflows
* remove hdf5

Author: JoschD

.github/workflows/README.md

@@ -6,22 +6,22 @@ They are organised as follows.
 ### Documentation
 
 The `documentation` workflow triggers on any push to master, builds the documentation and pushes it to the `gh-pages` branch (if the build is successful).
-It runs on `ubuntu-latest` and our lowest supported Python version, `Python 3.7`.
+It runs on `ubuntu-latest` and Python version, `Python 3.9`.
 
 ### Testing Suite
 
 Tests are ensured in the `tests` workflow, which triggers on all pushes.
-It runs on a matrix of all supported operating systems (ubuntu-18.04, ubuntu-20.04, windows-latest and macos-latest) for all supported Python versions (currently `3.7`, `3.8`, `3.9` and `3.10`).
+It runs on a matrix of all supported operating systems (ubuntu-20.04, ubuntu-22.04, windows-latest and macos-latest) for all supported Python versions (currently `3.8`, `3.9` and `3.10`).
 
 ### Test Coverage
 
 Test coverage is calculated in the `coverage` wokflow, which triggers on pushes to `master` and any push to a `pull request`.
-It runs on `ubuntu-latest` & the lowest supported Python version (`Python 3.7`), and reports the coverage results of the test suite to `CodeClimate`.
+It runs on `ubuntu-latest` & the Python version (`Python 3.9`), and reports the coverage results of the test suite to `CodeClimate`.
 
 ### Regular Testing
 
 A `cron` workflow triggers every Monday at 3am (UTC time) and runs the full testing suite, on all available operating systems and supported Python versions.
-It also runs on `Python 3.x` so that newly released Python versions that would break tests are automatically included.
+It also runs on `Python 3.x` so that newly released Python versions that would break tests are automatically detected.
 
 ### Publishing
 

.github/workflows/coverage.yml

@@ -16,9 +16,9 @@ jobs:
     name: ${{ matrix.os }} / ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
-      matrix:  # only lowest supported Python on latest ubuntu
+      matrix:
         os: [ubuntu-latest]
-        python-version: [3.7]
+        python-version: [3.9]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/cron.yml

@@ -15,18 +15,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04, ubuntu-20.04, macos-latest, windows-latest]
+        os: [ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest]
         # Make sure to escape 3.10 with quotes so it doesn't get interpreted as float 3.1 by GA's parser
-        python-version: [3.7, 3.8, 3.9, "3.10", 3.x]  # crons should always run latest python hence 3.x
-        exclude:
-          - os: windows-latest
-            python-version: 3.10  # jpype1 doesn't build there
-          # - os: macos-latest
-          #   python-version: 3.10  # jpype1 doesn't build there
-          - os: windows-latest
-            python-version: 3.10  # jpype1 doesn't build there
-          # - os: macos-latest
-          #   python-version: 3.10  # jpype1 doesn't build there
+        python-version: [3.8, 3.9, "3.10", "3.11", 3.x]  # crons should always run latest python hence 3.x
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/documentation.yml

@@ -1,11 +1,14 @@
-# Build and publish documentation
+# Build documentation
+# The build is uploaded as artifact if the triggering event is a push for a pull request
+# The build is published to github pages if the triggering event is a push to the master branch (PR merge)
 name: Build and upload documentation
 
 defaults:
   run:
     shell: bash
 
-on:  # Runs on any push event to master
+on:  # Runs on any push event in a PR or any push event to master
+  pull_request:
   push:
     branches:
       - 'master'
@@ -15,9 +18,9 @@ jobs:
     name: ${{ matrix.os }} / ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
-      matrix:  # only lowest supported Python on latest ubuntu
+      matrix:
         os: [ubuntu-latest]
-        python-version: [3.7]
+        python-version: [3.9]
 
     steps:
       - uses: actions/checkout@v3
@@ -42,10 +45,16 @@ jobs:
       - name: Build documentation
         run: python -m sphinx -b html doc ./doc_build -d ./doc_build
 
+      - name: Upload build artifacts  # upload artifacts so reviewers can have a quick look without building documentation from the branch locally
+        uses: actions/upload-artifact@v3
+        if: success() && github.event_name == 'pull_request'  # only for pushes in PR
+        with:
+          name: site-build
+          path: doc_build
+          retention-days: 5
+
       - name: Upload documentation to gh-pages
-        if: ${{ success() }}
-        uses: JamesIves/github-pages-deploy-action@3.6.2
+        if: success() && github.ref == 'refs/heads/master'  # only for pushes to master
+        uses: JamesIves/github-pages-deploy-action@v4
         with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BRANCH: gh-pages
-          FOLDER: doc_build
+          folder: doc_build

.github/workflows/publish.yml

@@ -14,9 +14,9 @@ jobs:
     name: ${{ matrix.os }} / ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
-      matrix:  # only lowest supported Python on ubuntu-latest
+      matrix:
         os: [ubuntu-latest]
-        python-version: [3.7]
+        python-version: [3.9]
 
 
     steps:

.github/workflows/tests.yml

@@ -5,25 +5,18 @@ defaults:
   run:
     shell: bash
 
-on:  # Runs on all push events to any branch that isn't master
-  push:
-    branches-ignore:
-      - 'master'
+on:  [push]  # Runs on all push events to any branch
+
 
 jobs:
   tests:
     name: ${{ matrix.os }} / ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04, ubuntu-20.04, macos-latest, windows-latest]
+        os: [ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest]
         # Make sure to escape 3.10 with quotes so it doesn't get interpreted as float 3.1 by GA's parser
-        python-version: [3.7, 3.8, 3.9, "3.10"]
-        exclude:
-          - os: windows-latest
-            python-version: 3.10  # jpype1 doesn't build there
-          # - os: macos-latest
-          #   python-version: 3.10  # jpype1 doesn't build there
+        python-version: [3.8, 3.9, "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3
@@ -46,4 +39,4 @@ jobs:
         run: python -m pip install '.[test]'
 
       - name: Run basic tests
-        run: python -m pytest -m "not cern_network" --mpl --cov-report xml --cov=pylhc
+        run: python -m pytest -m "not cern_network" --mpl

CHANGELOG.md

@@ -1,5 +1,16 @@
 # pylhc Changelog
 
+## Version 0.8.0
+
+Fixed:
+ - replaced `get_loc` with `get_indexer` to be compatible with `pandas` > 2.0
+
+Added:
+ - `beamprocess` option to Machine Settings Info, to allow manual override.
+
+Removed:
+ - `hdf5` reader/writer for `tfs` as this is already in `tfs-pandas`
+
 ## Version 0.7.4
 
 Changes in Machine Settings Info

doc/conf.py

@@ -114,7 +114,7 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

doc/entrypoints/analysis.rst

@@ -3,7 +3,9 @@ Analysis Tools
 
 .. automodule:: pylhc.bsrt_analysis
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.forced_da_analysis
     :members:
+    :noindex:

doc/entrypoints/bpm_calibration.rst

@@ -3,3 +3,4 @@ BPM Calibration
 
 .. automodule:: pylhc.bpm_calibration
     :members:
+    :noindex:

doc/entrypoints/kickgroups.rst

@@ -3,3 +3,4 @@ KickGroup Informations
 
 .. automodule:: pylhc.kickgroups
     :members:
+    :noindex:

doc/entrypoints/lsa_to_madx.rst

@@ -3,3 +3,4 @@ LSA Knobs to MAD-X Expressions Converter
 
 .. automodule:: pylhc.lsa_to_madx
     :members:
+    :noindex:

doc/entrypoints/machine_info.rst

@@ -3,8 +3,10 @@ Machine Information Tools
 
 .. automodule:: pylhc.machine_settings_info
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.bsrt_logger
     :members:
+    :noindex:
 

doc/modules/calibration.rst

@@ -3,8 +3,10 @@ BPM Calibration Utilities
 
 .. automodule:: pylhc.calibration.beta
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.calibration.dispersion
     :members:
+    :noindex:
 

doc/modules/constants.rst

@@ -4,19 +4,24 @@ Constants Definitions
 
 .. automodule:: pylhc.constants.general
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.constants.forced_da_analysis
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.constants.kickgroups
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.constants.machine_settings_info
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.constants.calibration
     :members:
+    :noindex:

doc/modules/data_extract.rst

@@ -3,8 +3,10 @@ Data Extraction
 
 .. automodule:: pylhc.data_extract.lsa
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc.data_extract.timber
     :members:
+    :noindex:
 

doc/modules/utils.rst

@@ -10,7 +10,7 @@
 __title__ = "pylhc"
 __description__ = "An accelerator physics script collection for the OMC team at CERN."
 __url__ = "https://github.com/pylhc/pylhc"
-__version__ = "0.7.4"
+__version__ = "0.8.0"
 __author__ = "pylhc"
 __author_email__ = "pylhc@github.com"
 __license__ = "MIT"

pylhc/__init__.py

@@ -22,11 +22,12 @@
 import pandas as pd
 import parse
 import pytz
+
 import tfs
 from generic_parser import EntryPointParameters, entrypoint
 from omc3.utils import logging_tools, time_tools
-
 from pylhc.constants.general import TFS_SUFFIX, TIME_COLUMN
+from pylhc.forced_da_analysis import get_approximate_index
 
 LOG = logging_tools.get_logger(__name__)
 PLOT_FILE_SUFFIX = ".pdf"
@@ -150,7 +151,7 @@ def _select_files(opt, files_df):
         [opt["starttime"], opt["endtime"]], ["first_valid_index", "last_valid_index"]
     ):
         indices.append(
-            _get_closest_index(files_df, time if time is not None else getattr(files_df, fct)())
+            get_approximate_index(files_df, time if time is not None else getattr(files_df, fct)())
         )
 
     return files_df.iloc[indices[0] : indices[1] + 1]
@@ -178,10 +179,6 @@ def _load_files_in_df(opt):
     return files_df
 
 
-def _get_closest_index(df, time):
-    return df.index.get_loc(time, method="nearest")
-
-
 def _get_timestamp_from_name(name, formatstring):
     year, month, day, hour, minute, second, microsecond = map(int, parse.parse(formatstring, name))
     return datetime.datetime(
@@ -203,12 +200,11 @@ def _load_pickled_data(opt, files_df):
     merged_df = pd.DataFrame()
     for bsrtfile in files_df["FILES"]:
         data = pickle.load(gzip.open(bsrtfile, "rb"))
-        for entry in data:
-            entry = _check_and_fix_entries(entry)
-            merged_df = merged_df.append(entry, ignore_index=True)
+        new_df = pd.DataFrame.from_records([_check_and_fix_entries(entry) for entry in data])
+        merged_df = pd.concat([merged_df, new_df], axis="index", ignore_index=True)
 
     merged_df = merged_df.set_index(
-        pd.to_datetime(merged_df["acqTime"], format=BSRT_FESA_TIME_FORMAT)
+        pd.to_datetime(merged_df["acqTime"], format=BSRT_FESA_TIME_FORMAT, utc=True)
     )
     merged_df.index.name = "TimeIndex"
     merged_df = merged_df.drop_duplicates(subset=["acqCounter", "acqTime"])
@@ -363,7 +359,7 @@ def plot_crosssection_for_timesteps(opt, bsrt_df):
     for idx, _ in kick_df.iterrows():
         timestamp = pd.to_datetime(time_tools.cern_utc_string_to_utc(idx))
 
-        data_row = bsrt_df.iloc[_get_closest_index(bsrt_df, timestamp)]
+        data_row = bsrt_df.iloc[get_approximate_index(bsrt_df, timestamp)]
         fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(18, 9), constrained_layout=True)
 
         fig.suptitle(f"Timestamp: {timestamp}")

pylhc/bsrt_analysis.py

@@ -175,7 +175,7 @@ def _get_factors_from_phase_fit(
         if calibration_phase_fit is None:
             calibration_phase_fit = c_fit
         else:
-            calibration_phase_fit = calibration_phase_fit.append(c_fit)
+            calibration_phase_fit = pd.concat([calibration_phase_fit, c_fit], axis="index")
 
     # Change the colum names for _fit
     calibration_phase_fit.columns = (LABELS[3], LABELS[4])

pylhc/calibration/beta.py

@@ -222,6 +222,6 @@ def get_calibration_factors_from_dispersion(
         if "X" not in calibration_factors.keys():
             calibration_factors = {"X": factors_for_ip}
         else:
-            calibration_factors["X"] = calibration_factors["X"].append(factors_for_ip)
+            calibration_factors["X"] = pd.concat([calibration_factors["X"], factors_for_ip], axis="index")
 
     return calibration_factors

pylhc/calibration/dispersion.py

@@ -203,7 +203,9 @@ def get_trim_history(
         LOG.debug(f"{len(trims)} trims extracted.")
         trims_not_found = [k for k in knobs if k not in trims.keys()]
         if len(trims_not_found):
-            LOG.warning(f"The following knobs were not found in '{beamprocess}': {trims_not_found}")
+            LOG.warning(
+                f"The following knobs were not found in '{beamprocess}' "
+                f"or had no trims during the given time: {trims_not_found}")
         return trims
 
     def get_beamprocess_info(self, beamprocess: Union[str, object]) -> Dict: