From 3b3ea298229759ed360d493dd4ae0fb2fcb079a8 Mon Sep 17 00:00:00 2001
From: David Joy <10147749+dmjoy@users.noreply.github.com>
Date: Tue, 16 Apr 2024 19:56:21 -0400
Subject: [PATCH] Fix some non-determinism in choice ordering for Kaleido

---
 align_system/algorithms/kaleido_adm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/align_system/algorithms/kaleido_adm.py b/align_system/algorithms/kaleido_adm.py
index d5cebe5c..c86598c4 100644
--- a/align_system/algorithms/kaleido_adm.py
+++ b/align_system/algorithms/kaleido_adm.py
@@ -102,7 +102,7 @@ def estimate_kdma_values(self,
 
         rows = []
         for choice in choices:
-            other_choices_str = ', '.join(['"{}"'.format(c) for c in (set(choices) - {choice})])
+            other_choices_str = ', '.join(['"{}"'.format(c) for c in choices if c != choice])
             choice_prompt = format_template(
                 prompt_template,
                 allow_extraneous=True,