From 87ea37047cba292203d26c57147f206ec6357bf9 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Wed, 10 Jul 2024 13:47:37 -0400 Subject: [PATCH 1/9] add error message --- src/spey_pyhf/data.py | 46 +++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/spey_pyhf/data.py b/src/spey_pyhf/data.py index 5770f07..fad90d4 100644 --- a/src/spey_pyhf/data.py +++ b/src/spey_pyhf/data.py @@ -1,14 +1,16 @@ -from typing import Optional, List, Tuple, Dict, Text, Union, Iterator - -from dataclasses import dataclass +import copy +import json +import os from abc import ABC, abstractmethod -import json, copy, os -import numpy as np +from dataclasses import dataclass +from typing import Dict, Iterator, List, Optional, Text, Tuple, Union -from spey.base import ModelConfig +import numpy as np from spey import ExpectationType +from spey.base import ModelConfig +from spey.system.exceptions import InvalidInput -from . import manager, WorkspaceInterpreter +from . import WorkspaceInterpreter, manager class Base(ABC): @@ -289,13 +291,31 @@ def __call__( ) if expected == ExpectationType.apriori: - data = sum( - ( - self.expected_background_yields[ch] + try: + data = sum( + ( + self.expected_background_yields[ch] + for ch in self._model.config.channels + ), + [], + ) + except KeyError as err: + # provide a useful error message to guide the user to the solution + missing_channels = [ + ch for ch in self._model.config.channels - ), - [], - ) + if ch not in self.expected_background_yields + ] + raise InvalidInput( + "Unable to construct expected data. " + + (len(missing_channels) > 0) + * ( + "\nThis is likely due to missing channels in the signal patch. " + + "The missing channels are: " + + ", ".join(missing_channels) + + "\nPlease provide appropriate action for the missing channels to continue." + ) + ) from err if include_aux: data += self._model.config.auxdata else: From e3711dfbc4da552562194d9526e2028e39fb14ae Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Wed, 10 Jul 2024 13:51:15 -0400 Subject: [PATCH 2/9] bump version --- .zenodo.json | 6 +++--- src/spey_pyhf/_version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index f72d4d1..c3c3bd5 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,8 +1,8 @@ { "description": "pyhf plug-in for spey package", "license": "MIT", - "title": "SpeysideHEP/spey-pyhf: v0.1.4", - "version": "v0.1.4", + "title": "SpeysideHEP/spey-pyhf: v0.1.5", + "version": "v0.1.5", "upload_type": "software", "creators": [ { @@ -29,7 +29,7 @@ }, { "scheme": "url", - "identifier": "https://github.com/SpeysideHEP/spey-pyhf/tree/v0.1.4", + "identifier": "https://github.com/SpeysideHEP/spey-pyhf/tree/v0.1.5", "relation": "isSupplementTo" }, { diff --git a/src/spey_pyhf/_version.py b/src/spey_pyhf/_version.py index c692f32..ee554de 100644 --- a/src/spey_pyhf/_version.py +++ b/src/spey_pyhf/_version.py @@ -1,3 +1,3 @@ """Version of the spey - pyhf plugin""" -__version__ = "0.1.4" +__version__ = "0.1.5-beta" From 1218951679ae32c23930cb7479838f1eb8de7f96 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Wed, 10 Jul 2024 13:51:23 -0400 Subject: [PATCH 3/9] update changelog --- docs/releases/changelog-v0.1.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/releases/changelog-v0.1.md b/docs/releases/changelog-v0.1.md index 8521111..d5c09c8 100644 --- a/docs/releases/changelog-v0.1.md +++ b/docs/releases/changelog-v0.1.md @@ -30,6 +30,9 @@ [#5](https://github.com/SpeysideHEP/spey-pyhf/issues/5). ([#2](https://github.com/SpeysideHEP/spey-pyhf/pull/2)) +* Add an error message for incomplete patch set + ([#12](https://github.com/SpeysideHEP/spey-pyhf/pull/12)) + ## Contributors This release contains contributions from (in alphabetical order): From e9b7b82d2d5f61ab9536873d039719036a35cde1 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Thu, 11 Jul 2024 11:20:09 -0400 Subject: [PATCH 4/9] undefined channel handling * added separate remove list to remove channels if they are specifically set as remove, otherwise they will be included in the model --- src/spey_pyhf/helper_functions.py | 83 ++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/src/spey_pyhf/helper_functions.py b/src/spey_pyhf/helper_functions.py index 07049a5..eff19a8 100644 --- a/src/spey_pyhf/helper_functions.py +++ b/src/spey_pyhf/helper_functions.py @@ -1,5 +1,6 @@ """Helper function for creating and interpreting pyhf inputs""" -from typing import Dict, Iterator, List, Text, Union, Optional +import logging +from typing import Dict, Iterator, List, Optional, Text, Tuple, Union __all__ = ["WorkspaceInterpreter"] @@ -8,7 +9,10 @@ def __dir__(): return __all__ -def remove_from_json(idx: int) -> Dict: +log = logging.getLogger("Spey") + + +def remove_from_json(idx: int) -> Dict[Text, Text]: """ Remove channel from the json file @@ -16,7 +20,7 @@ def remove_from_json(idx: int) -> Dict: idx (``int``): index of the channel Returns: - ``Dict``: + ``Dict[Text, Text]``: JSON patch """ return {"op": "remove", "path": f"/channels/{idx}"} @@ -59,13 +63,19 @@ class WorkspaceInterpreter: background_only_model (``Dict``): descrioption for the background only statistical model """ - __slots__ = ["background_only_model", "_signal_dict", "_signal_modifiers"] + __slots__ = [ + "background_only_model", + "_signal_dict", + "_signal_modifiers", + "_to_remove", + ] def __init__(self, background_only_model: Dict): self.background_only_model = background_only_model """Background only statistical model description""" self._signal_dict = {} self._signal_modifiers = {} + self._to_remove = [] def __getitem__(self, item): return self.background_only_model[item] @@ -89,8 +99,9 @@ def bin_map(self) -> Dict[Text, int]: def expected_background_yields(self) -> Dict[Text, List[float]]: """Retreive expected background yields with respect to signal injection""" yields = {} + undefined_channels = [] for channel in self["channels"]: - if channel["name"] in self._signal_dict: + if channel["name"] not in self.remove_list: yields[channel["name"]] = [] for smp in channel["samples"]: if len(yields[channel["name"]]) == 0: @@ -98,6 +109,19 @@ def expected_background_yields(self) -> Dict[Text, List[float]]: yields[channel["name"]] = [ ch + dt for ch, dt in zip(yields[channel["name"]], smp["data"]) ] + if channel["name"] not in self._signal_dict: + undefined_channels.append(channel["name"]) + if len(undefined_channels) > 0: + log.warning( + "Some of the channels are not defined in the patch set, " + "these channels will be kept in the statistical model. " + ) + log.warning( + "If these channels are meant to be removed, please indicate them in the patch set." + ) + log.warning( + "Please check the following channel(s): " + ", ".join(undefined_channels) + ) return yields def guess_channel_type(self, channel_name: Text) -> Text: @@ -197,8 +221,10 @@ def make_patch(self) -> List[Dict]: ich, self._signal_dict[channel], self._signal_modifiers[channel] ) ) - else: + elif channel in self._to_remove: to_remove.append(remove_from_json(ich)) + else: + log.warning(f"Undefined channel in the patch set: {channel}") to_remove.sort(key=lambda p: p["path"].split("/")[-1], reverse=True) @@ -207,12 +233,39 @@ def make_patch(self) -> List[Dict]: def reset_signal(self) -> None: """Clear the signal map""" self._signal_dict = {} + self._to_remove = [] def add_patch(self, signal_patch: List[Dict]) -> None: """Inject signal patch""" - self._signal_dict = self.patch_to_map(signal_patch=signal_patch) + self._signal_dict, self._to_remove = self.patch_to_map( + signal_patch=signal_patch, return_remove_list=True + ) - def patch_to_map(self, signal_patch: List[Dict]) -> Dict[Text, Dict]: + def remove_channel(self, channel_name: Text) -> None: + """ + Remove channel from the likelihood + + Args: + channel_name (``Text``): name of the channel to be removed + """ + if channel_name in self.channels: + if channel_name not in self._to_remove: + self._to_remove.append(channel_name) + else: + log.error( + f"Channel {channel_name} does not exist in the background only model. " + + "The available channels are " + + ", ".join(list(self.channels)) + ) + + @property + def remove_list(self) -> List[Text]: + """Channels to be removed from the model""" + return self._to_remove + + def patch_to_map( + self, signal_patch: List[Dict], return_remove_list: bool = False + ) -> Union[Tuple[Dict[Text, Dict], List[Text]], Dict[Text, Dict]]: """ Convert JSONPatch into signal map @@ -223,20 +276,26 @@ def patch_to_map(self, signal_patch: List[Dict]) -> Dict[Text, Dict]: Args: signal_patch (``List[Dict]``): JSONPatch for the signal + return_remove_list (``bool``): Inclure channels to be removed in the output Returns: - ``Dict[Text, Dict]``: - signal map including the data and modifiers + ``Tuple[Dict[Text, Dict], List[Text]]`` or ``Dict[Text, Dict]``: + signal map including the data and modifiers and the list of channels to be removed. """ signal_map = {} + to_remove = [] for item in signal_patch: + path = int(item["path"].split("/")[2]) + channel_name = self["channels"][path]["name"] if item["op"] == "add": - path = int(item["path"].split("/")[2]) - channel_name = self["channels"][path]["name"] signal_map[channel_name] = { "data": item["value"]["data"], "modifiers": item["value"].get( "modifiers", _default_modifiers(poi_name=self.poi_name[0][1]) ), } + elif item["op"] == "remove": + to_remove.append(channel_name) + if return_remove_list: + return signal_map, to_remove return signal_map From 366c9a50432902c61beea8d093ff1f7fbd632859 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Thu, 11 Jul 2024 11:20:24 -0400 Subject: [PATCH 5/9] bump spey requirement --- setup.py | 2 +- src/spey_pyhf/interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index dae112e..b59c4dd 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("src/spey_pyhf/_version.py", mode="r", encoding="UTF-8") as f: version = f.readlines()[-1].split()[-1].strip("\"'") -requirements = ["pyhf==0.7.6", "spey>=0.1.5"] +requirements = ["pyhf==0.7.6", "spey>=0.1.9"] docs = [ "sphinx==6.2.1", diff --git a/src/spey_pyhf/interface.py b/src/spey_pyhf/interface.py index e39a5e7..868d3db 100644 --- a/src/spey_pyhf/interface.py +++ b/src/spey_pyhf/interface.py @@ -73,7 +73,7 @@ class PyhfInterface(BackendBase): """Version of the backend""" author: Text = "SpeysideHEP" """Author of the backend""" - spey_requires: Text = ">=0.1.5,<0.2.0" + spey_requires: Text = ">=0.1.9,<0.2.0" """Spey version required for the backend""" doi: List[Text] = ["10.5281/zenodo.1169739", "10.21105/joss.02823"] """Citable DOI for the backend""" From be9de8264ae2b72a9fab2c7f3fc3e96f9d1821a9 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Thu, 11 Jul 2024 11:21:16 -0400 Subject: [PATCH 6/9] update changelog --- docs/releases/changelog-v0.1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/releases/changelog-v0.1.md b/docs/releases/changelog-v0.1.md index d5c09c8..dce9b3f 100644 --- a/docs/releases/changelog-v0.1.md +++ b/docs/releases/changelog-v0.1.md @@ -21,6 +21,9 @@ * Model loading has been improved for prefit and postfit scenarios. ([#10](https://github.com/SpeysideHEP/spey-pyhf/pull/10)) +* Improve undefined channel handling in the patchset + ([#12](https://github.com/SpeysideHEP/spey-pyhf/pull/12)) + ## Bug fixes * Bugfix in `simplify` module, where signal injector was not initiated properly. @@ -30,9 +33,6 @@ [#5](https://github.com/SpeysideHEP/spey-pyhf/issues/5). ([#2](https://github.com/SpeysideHEP/spey-pyhf/pull/2)) -* Add an error message for incomplete patch set - ([#12](https://github.com/SpeysideHEP/spey-pyhf/pull/12)) - ## Contributors This release contains contributions from (in alphabetical order): From 8bd98ecc7ca9a71d2fd7cf808a599990b4de4280 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Thu, 11 Jul 2024 11:54:05 -0400 Subject: [PATCH 7/9] update docs --- docs/tutorials/utils.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/utils.md b/docs/tutorials/utils.md index 6b4e42f..bccc338 100644 --- a/docs/tutorials/utils.md +++ b/docs/tutorials/utils.md @@ -65,7 +65,7 @@ we can inject signal to any channel we like ````{margin} ```{admonition} Attention! :class: attention - Notice that the rest of the channels will be removed. If some of the channels are needed during the inference, simply remove the ones with `"op": "remove"` tag from the patch set. Patch set can be generated via `interpreter.make_patch()` function. + Notice that the rest of the channels will be added without any signal yields. If some of these channels need to be removed from the patch set, they can be added to the remove list via the ``remove_channel()`` function. **Note:** This behaviour has been updated in ``v0.1.5``. In the older versions, the channels that were not declared were removed. ``` ```` @@ -73,7 +73,7 @@ we can inject signal to any channel we like interpreter.inject_signal('SRHMEM_mct2', [5.0, 12.0, 4.0]) ``` -Notice that I only added 3 inputs since the `"SRHMEM_mct2"` region has only 3 bins. One can inject signals to as many channels as one wants, but for simplicity, we will use only one channel. Now we are ready to export this signal patch and compute the exclusion limit +Notice that we only added 3 inputs since the `"SRHMEM_mct2"` region has only 3 bins. One can inject signals to as many channels as one wants, but for simplicity, we will use only one channel. Now we are ready to export this signal patch and compute the exclusion limit ```{code-cell} ipython3 pdf_wrapper = spey.get_backend("pyhf") From 4e5a02e5e7338000a003f5c6cdec1e0a6297e8c1 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Thu, 11 Jul 2024 12:56:39 -0400 Subject: [PATCH 8/9] update documentation --- src/spey_pyhf/helper_functions.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/spey_pyhf/helper_functions.py b/src/spey_pyhf/helper_functions.py index eff19a8..ba9c88e 100644 --- a/src/spey_pyhf/helper_functions.py +++ b/src/spey_pyhf/helper_functions.py @@ -245,6 +245,8 @@ def remove_channel(self, channel_name: Text) -> None: """ Remove channel from the likelihood + .. versionadded:: 0.1.5 + Args: channel_name (``Text``): name of the channel to be removed """ @@ -260,7 +262,12 @@ def remove_channel(self, channel_name: Text) -> None: @property def remove_list(self) -> List[Text]: - """Channels to be removed from the model""" + """ + Channels to be removed from the model + + .. versionadded:: 0.1.5 + + """ return self._to_remove def patch_to_map( @@ -276,7 +283,9 @@ def patch_to_map( Args: signal_patch (``List[Dict]``): JSONPatch for the signal - return_remove_list (``bool``): Inclure channels to be removed in the output + return_remove_list (``bool``, default ``False``): Inclure channels to be removed in the output + + .. versionadded:: 0.1.5 Returns: ``Tuple[Dict[Text, Dict], List[Text]]`` or ``Dict[Text, Dict]``: From 125d62e3b698b0be6f862126bf7898232431368f Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Fri, 12 Jul 2024 11:47:18 -0400 Subject: [PATCH 9/9] remove beta --- src/spey_pyhf/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spey_pyhf/_version.py b/src/spey_pyhf/_version.py index ee554de..51ce7ae 100644 --- a/src/spey_pyhf/_version.py +++ b/src/spey_pyhf/_version.py @@ -1,3 +1,3 @@ """Version of the spey - pyhf plugin""" -__version__ = "0.1.5-beta" +__version__ = "0.1.5"