Merge pull request #42 from ITM-Kitware/dev/metrics-eval-fix-missing-…

…justification Dev/metrics eval fix missing justification
ITM-Kitware · Mar 22, 2024 · 13fbc97 · 13fbc97
2 parents eafbea3 + fae9a24
commit 13fbc97
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,12 @@
 This changelog follows the specifications detailed in: [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), although we have not yet reached a `1.0.0` release.
 
+## 0.3.2
+
+### Fixed
+
+* Fixed issue where justifications weren't being populated for both Llama2SingleKDMAADM and the HybridKaleidoADM
+
 ## 0.3.1
 
 ### Added

diff --git a/align_system/algorithms/kaleido_adm.py b/align_system/algorithms/kaleido_adm.py
@@ -296,4 +296,7 @@ def choose_action(self, scenario_state, available_actions, alignment_target, **k
             choices_unstructured,
             distance_fn=kwargs.get('distance_fn', DefaultDistanceFunction))
 
-        return available_actions[selected_choice_idx]
+        action_to_take = available_actions[selected_choice_idx]
+        action_to_take.justification = kaleido_results.loc[selected_choice_idx, :].explanation
+
+        return action_to_take
diff --git a/align_system/algorithms/llama_2_single_kdma_adm.py b/align_system/algorithms/llama_2_single_kdma_adm.py
@@ -873,6 +873,7 @@ def choose_action(self, scenario_state, available_actions, alignment_target, **k
             **kwargs)
 
         action_to_take = available_actions[response['choice']]
+        action_to_take.justification = response['info'].get('reasoning')
 
         if action_to_take.action_type == ActionTypeEnum.APPLY_TREATMENT:
             # If the additional required fields are already populated

diff --git a/align_system/cli/run_align_system.py b/align_system/cli/run_align_system.py
@@ -252,7 +252,12 @@ def run_action_based_chat_system(interface,
             if state_has_changed:
                 noop_actions = []
             else:
-                noop_actions.append(action_to_take)
+                # Strip out the justification string (provided by our
+                # ADMs) from no-op actions so that it can be compared
+                # to the original actions
+                _tmp_action_to_take = deepcopy(action_to_take)
+                _tmp_action_to_take.justification = None
+                noop_actions.append(_tmp_action_to_take)
 
             scenario_complete = current_state.scenario_complete
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "align-system"
-version = "0.3.1"
+version = "0.3.2"
 description = ""
 authors = ["David Joy <10147749+dmjoy@users.noreply.github.com>"]
 readme = "README.md"