From 5f9f3e28cd72a0e67614d24eb15e461da07360f9 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kaiyutony@gmail.com>
Date: Fri, 23 Feb 2024 19:04:24 +0000
Subject: [PATCH 01/78] dev version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index bd28e85c..2db1ad63 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pomdp-py"
-version = "1.3.4"
+version = "1.3.5-dev"
 description = "Python POMDP Library."
 authors = [{name = "Kaiyu Zheng", email = "kzheng10@cs.brown.edu"}]
 license = {file = "LICENSE"}

From c8f8233a34bcd7e7769aaa499937a129e9548412 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 23 Feb 2024 14:26:28 -0500
Subject: [PATCH 02/78] adding DictState

---
 pomdp_py/framework/oopomdp.pyx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pomdp_py/framework/oopomdp.pyx b/pomdp_py/framework/oopomdp.pyx
index 5dc5172c..433ff960 100644
--- a/pomdp_py/framework/oopomdp.pyx
+++ b/pomdp_py/framework/oopomdp.pyx
@@ -405,3 +405,8 @@ cdef class OOBelief(GenerativeDistribution):
     def b(self, objid):
         """convenient alias function call"""
         return self._object_beliefs[objid]
+
+cdef class DictState(ObjectState):
+    """This is synonymous as ObjectState, but does not convey 'objectness'
+    of the information being described."""
+    pass

From 73c601f3ae5e39895e836eb41bb061b369711c15 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Thu, 7 Mar 2024 15:19:58 -0500
Subject: [PATCH 03/78] run ci on dev-latest branch

---
 .github/workflows/macos.yml          | 4 ++--
 .github/workflows/pre_commit.yml     | 2 +-
 .github/workflows/python-package.yml | 4 ++--
 .github/workflows/ubuntu.yml         | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index cf046b3f..38fdf8f1 100644
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -2,9 +2,9 @@ name: pomdp-py repo macOS build
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, dev-latest ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, dev-latest ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
diff --git a/.github/workflows/pre_commit.yml b/.github/workflows/pre_commit.yml
index 8cb7ed2c..b0029f92 100644
--- a/.github/workflows/pre_commit.yml
+++ b/.github/workflows/pre_commit.yml
@@ -7,7 +7,7 @@ on:
   pull_request:
     branches:
       - main
-
+      - dev-latest
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 15c55d0e..83218e23 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -5,9 +5,9 @@ name: pomdp-py package
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "dev-latest" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "dev-latest" ]
 
 jobs:
   build:
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index 42bad4e1..b30e0d6e 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -2,9 +2,9 @@ name: pomdp-py repo ubuntu build
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, dev-latest ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, dev-latest ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

From a7fb3e54318ba1c233ea454b22bb8443097cd1d8 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 23 Feb 2024 16:35:26 -0500
Subject: [PATCH 04/78] DictState doesn't need objclass

---
 pomdp_py/framework/oopomdp.pxd | 3 +++
 pomdp_py/framework/oopomdp.pyx | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/pomdp_py/framework/oopomdp.pxd b/pomdp_py/framework/oopomdp.pxd
index 2392a1e8..320842c0 100644
--- a/pomdp_py/framework/oopomdp.pxd
+++ b/pomdp_py/framework/oopomdp.pxd
@@ -25,3 +25,6 @@ cdef class OOObservationModel(ObservationModel):
 
 cdef class OOBelief(GenerativeDistribution):
     cdef dict _object_beliefs
+
+cdef class DictState(ObjectState):
+    pass
diff --git a/pomdp_py/framework/oopomdp.pyx b/pomdp_py/framework/oopomdp.pyx
index 433ff960..020772c5 100644
--- a/pomdp_py/framework/oopomdp.pyx
+++ b/pomdp_py/framework/oopomdp.pyx
@@ -409,4 +409,5 @@ cdef class OOBelief(GenerativeDistribution):
 cdef class DictState(ObjectState):
     """This is synonymous as ObjectState, but does not convey 'objectness'
     of the information being described."""
-    pass
+    def __init__(self, attributes):
+        super().__init__(None, attributes)

From a3c66e31ca9e4c0752c62ac1cc0574730f49be74 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 23 Feb 2024 16:35:38 -0500
Subject: [PATCH 05/78] allow Agent and Environment to be created without all
 models

---
 pomdp_py/framework/basics.pyx  | 31 +++++++++++++++++++++----------
 tests/test_framework_basics.py | 15 +++++++++++++++
 2 files changed, 36 insertions(+), 10 deletions(-)
 create mode 100644 tests/test_framework_basics.py

diff --git a/pomdp_py/framework/basics.pyx b/pomdp_py/framework/basics.pyx
index 478f58ce..8df4eabf 100644
--- a/pomdp_py/framework/basics.pyx
+++ b/pomdp_py/framework/basics.pyx
@@ -345,11 +345,6 @@ cdef class Agent:
         self._observation_model = observation_model
         self._reward_model = reward_model
         self._blackbox_model = blackbox_model
-        # It cannot be the case that both explicit models and blackbox model are None.
-        if self._blackbox_model is None:
-            assert self._transition_model is not None\
-                and self._observation_model is not None\
-                and self._reward_model is not None
 
         # For online planning
         self._cur_belief = init_belief
@@ -419,6 +414,18 @@ cdef class Agent:
     def generative_model(self):
         return self.blackbox_model
 
+    def set_models(self, transition_model=None, observation_model=None,
+                   reward_model=None, blackbox_model=None):
+        """Re-assign the models to be the ones given."""
+        if transition_model is not None:
+            self._transition_model = transition_model
+        if observation_model is not None:
+            self._observation_model = observation_model
+        if reward_model is not None:
+            self._reward_model = reward_model
+        if blackbox_model is not None:
+            self._blackbox_model = blackbox_model
+
     def add_attr(self, attr_name, attr_value):
         """
         add_attr(self, attr_name, attr_value)
@@ -482,11 +489,6 @@ cdef class Environment:
         self._reward_model = reward_model
         self._blackbox_model = blackbox_model
 
-        # It cannot be the case that both explicit models and blackbox model are None.
-        if self._blackbox_model is None:
-            assert self._transition_model is not None\
-                and self._reward_model is not None
-
     @property
     def state(self):
         """Synonym for :meth:`cur_state`"""
@@ -512,6 +514,15 @@ cdef class Environment:
         """The :class:`BlackboxModel` underlying the environment"""
         return self._blackbox_model
 
+    def set_models(self, transition_model=None, reward_model=None, blackbox_model=None):
+        """Re-assign the models to be the ones given."""
+        if transition_model is not None:
+            self._transition_model = transition_model
+        if reward_model is not None:
+            self._reward_model = reward_model
+        if blackbox_model is not None:
+            self._blackbox_model = blackbox_model
+
     def state_transition(self, action, execute=True, discount_factor=1.0):
         """
         state_transition(self, action, execute=True)
diff --git a/tests/test_framework_basics.py b/tests/test_framework_basics.py
new file mode 100644
index 00000000..33a6fb94
--- /dev/null
+++ b/tests/test_framework_basics.py
@@ -0,0 +1,15 @@
+import pomdp_py
+
+description = "testing framework basics"
+
+def test_agent() -> None:
+    b0 = pomdp_py.Histogram({"hungry": 0.5, "full": 0.5})
+    policy = pomdp_py.UniformPolicyModel(["eat", "sleep"])
+    agent = pomdp_py.Agent(b0, policy)
+    assert agent.policy_model.sample(b0.random()) in ["eat", "sleep"]
+
+def run():
+    test_agent()
+
+if __name__ == "__main__":
+    run()

From 07628dabb32eb2393414d49b17f806d9a2a28369 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 23 Feb 2024 16:43:49 -0500
Subject: [PATCH 06/78] policy model can be optional upon init for Agent

---
 pomdp_py/framework/basics.pyx | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pomdp_py/framework/basics.pyx b/pomdp_py/framework/basics.pyx
index 8df4eabf..92acf905 100644
--- a/pomdp_py/framework/basics.pyx
+++ b/pomdp_py/framework/basics.pyx
@@ -333,7 +333,7 @@ cdef class Agent:
              blackbox_model=None)
     """
     def __init__(self, init_belief,
-                 policy_model,
+                 policy_model=None,
                  transition_model=None,
                  observation_model=None,
                  reward_model=None,
@@ -415,7 +415,7 @@ cdef class Agent:
         return self.blackbox_model
 
     def set_models(self, transition_model=None, observation_model=None,
-                   reward_model=None, blackbox_model=None):
+                   reward_model=None, blackbox_model=None, policy_model=None):
         """Re-assign the models to be the ones given."""
         if transition_model is not None:
             self._transition_model = transition_model
@@ -425,6 +425,8 @@ cdef class Agent:
             self._reward_model = reward_model
         if blackbox_model is not None:
             self._blackbox_model = blackbox_model
+        if policy_model is not None:
+            self._policy_model = policy_model
 
     def add_attr(self, attr_name, attr_value):
         """

From 439580b2650f3e979905f6d18524aace5bfa452b Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 23 Feb 2024 17:02:07 -0500
Subject: [PATCH 07/78] add test for setting model for agent

---
 tests/test_framework_basics.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tests/test_framework_basics.py b/tests/test_framework_basics.py
index 33a6fb94..1ff293f1 100644
--- a/tests/test_framework_basics.py
+++ b/tests/test_framework_basics.py
@@ -5,9 +5,34 @@
 def test_agent() -> None:
     b0 = pomdp_py.Histogram({"hungry": 0.5, "full": 0.5})
     policy = pomdp_py.UniformPolicyModel(["eat", "sleep"])
+
+    # test that agent can be created with incomplete models
     agent = pomdp_py.Agent(b0, policy)
     assert agent.policy_model.sample(b0.random()) in ["eat", "sleep"]
 
+    # test that
+    transition = pomdp_py.TabularTransitionModel(
+        {
+            # state, action, next_state
+            ("hungry", "eat", "full"): 0.7,
+            ("hungry", "eat", "hungry"): 0.3,
+
+            ("hungry", "sleep", "full"): 0.01,
+            ("hungry", "sleep", "hungry"): 0.99,
+
+            ("full", "eat", "full"): 0.9,
+            ("full", "eat", "hungry"): 0.1,
+
+            ("full", "sleep", "full"): 0.5,
+            ("full", "sleep", "hungry"): 0.5,
+        }
+    )
+    agent.set_models(transition_model=transition)
+    # next_state, state, action
+    assert agent.transition_model.probability("full", "hungry", "eat") == 0.7
+
+
+
 def run():
     test_agent()
 

From 779cc65bffae47614722b03c74b98b1833ec2c37 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 23 Feb 2024 17:06:12 -0500
Subject: [PATCH 08/78] test agent env set model

---
 tests/test_framework_basics.py | 57 ++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 23 deletions(-)

diff --git a/tests/test_framework_basics.py b/tests/test_framework_basics.py
index 1ff293f1..e3b4ebc5 100644
--- a/tests/test_framework_basics.py
+++ b/tests/test_framework_basics.py
@@ -2,39 +2,50 @@
 
 description = "testing framework basics"
 
-def test_agent() -> None:
-    b0 = pomdp_py.Histogram({"hungry": 0.5, "full": 0.5})
-    policy = pomdp_py.UniformPolicyModel(["eat", "sleep"])
+TRANSITION = pomdp_py.TabularTransitionModel(
+    {
+        # state, action, next_state
+        ("hungry", "eat", "full"): 0.7,
+        ("hungry", "eat", "hungry"): 0.3,
 
-    # test that agent can be created with incomplete models
-    agent = pomdp_py.Agent(b0, policy)
-    assert agent.policy_model.sample(b0.random()) in ["eat", "sleep"]
+        ("hungry", "sleep", "full"): 0.01,
+        ("hungry", "sleep", "hungry"): 0.99,
+
+        ("full", "eat", "full"): 0.9,
+        ("full", "eat", "hungry"): 0.1,
 
-    # test that
-    transition = pomdp_py.TabularTransitionModel(
-        {
-            # state, action, next_state
-            ("hungry", "eat", "full"): 0.7,
-            ("hungry", "eat", "hungry"): 0.3,
+        ("full", "sleep", "full"): 0.5,
+        ("full", "sleep", "hungry"): 0.5,
+    }
+)
 
-            ("hungry", "sleep", "full"): 0.01,
-            ("hungry", "sleep", "hungry"): 0.99,
+def test_agent_set_model() -> None:
+    b0 = pomdp_py.Histogram({"hungry": 0.5, "full": 0.5})
 
-            ("full", "eat", "full"): 0.9,
-            ("full", "eat", "hungry"): 0.1,
+    # test that agent can be created with incomplete models
+    # and we can set the agent's model after its creation
+    agent = pomdp_py.Agent(b0)
 
-            ("full", "sleep", "full"): 0.5,
-            ("full", "sleep", "hungry"): 0.5,
-        }
-    )
-    agent.set_models(transition_model=transition)
+    agent.set_models(transition_model=TRANSITION)
     # next_state, state, action
     assert agent.transition_model.probability("full", "hungry", "eat") == 0.7
 
+    policy = pomdp_py.UniformPolicyModel(["eat", "sleep"])
+    agent.set_models(policy_model=policy)
+    assert agent.policy_model.sample(b0.random()) in ["eat", "sleep"]
+
+def test_env_set_model() -> None:
+    # test that agent can be created with incomplete models
+    # and we can set the agent's model after its creation
+    env = pomdp_py.Environment(pomdp_py.SimpleState("hungry"))
+    env.set_models(transition_model=TRANSITION)
+    # next_state, state, action
+    assert env.transition_model.probability("full", "hungry", "eat") == 0.7
 
 
-def run():
-    test_agent()
+def run() -> None:
+    test_agent_set_model()
+    test_env_set_model()
 
 if __name__ == "__main__":
     run()

From 0200f41cd20c476950717a327fc98d16218001bc Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 23 Feb 2024 17:12:24 -0500
Subject: [PATCH 09/78] black

---
 tests/test_framework_basics.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_framework_basics.py b/tests/test_framework_basics.py
index e3b4ebc5..a99c0abe 100644
--- a/tests/test_framework_basics.py
+++ b/tests/test_framework_basics.py
@@ -7,18 +7,16 @@
         # state, action, next_state
         ("hungry", "eat", "full"): 0.7,
         ("hungry", "eat", "hungry"): 0.3,
-
         ("hungry", "sleep", "full"): 0.01,
         ("hungry", "sleep", "hungry"): 0.99,
-
         ("full", "eat", "full"): 0.9,
         ("full", "eat", "hungry"): 0.1,
-
         ("full", "sleep", "full"): 0.5,
         ("full", "sleep", "hungry"): 0.5,
     }
 )
 
+
 def test_agent_set_model() -> None:
     b0 = pomdp_py.Histogram({"hungry": 0.5, "full": 0.5})
 
@@ -34,6 +32,7 @@ def test_agent_set_model() -> None:
     agent.set_models(policy_model=policy)
     assert agent.policy_model.sample(b0.random()) in ["eat", "sleep"]
 
+
 def test_env_set_model() -> None:
     # test that agent can be created with incomplete models
     # and we can set the agent's model after its creation
@@ -47,5 +46,6 @@ def run() -> None:
     test_agent_set_model()
     test_env_set_model()
 
+
 if __name__ == "__main__":
     run()

From 736408196e5a5a703cedfbd016f6afbb6390a636 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 26 Feb 2024 09:25:13 -0500
Subject: [PATCH 10/78] option function names noun to verb

---
 pomdp_py/framework/basics.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pomdp_py/framework/basics.pyx b/pomdp_py/framework/basics.pyx
index 92acf905..8a803d85 100644
--- a/pomdp_py/framework/basics.pyx
+++ b/pomdp_py/framework/basics.pyx
@@ -587,13 +587,13 @@ cdef class Option(Action):
     Described in `Between MDPs and semi-MDPs:
     A framework for temporal abstraction in reinforcement learning`
     """
-    def initiation(self, state):
+    def instantiate(self, state):
         """
         initiation(self, state)
         Returns True if the given parameters satisfy the initiation set"""
         raise NotImplementedError
 
-    def termination(self, state):
+    def terminate(self, state):
         """termination(self, state)
         Returns a boolean of whether state satisfies the termination
         condition; Technically returning a float between 0 and 1 is also allowed."""

From e5359160717b2519f8e9347ddea222bc046b0769 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 26 Feb 2024 11:05:16 -0500
Subject: [PATCH 11/78] oops word correction

---
 pomdp_py/framework/basics.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pomdp_py/framework/basics.pyx b/pomdp_py/framework/basics.pyx
index 8a803d85..0640c16d 100644
--- a/pomdp_py/framework/basics.pyx
+++ b/pomdp_py/framework/basics.pyx
@@ -587,7 +587,7 @@ cdef class Option(Action):
     Described in `Between MDPs and semi-MDPs:
     A framework for temporal abstraction in reinforcement learning`
     """
-    def instantiate(self, state):
+    def initiate(self, state):
         """
         initiation(self, state)
         Returns True if the given parameters satisfy the initiation set"""

From 379f3d2e1f13f855a317361a4e8abf544ccab1fa Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 10:52:56 -0500
Subject: [PATCH 12/78] hashable frozen weighted particles

---
 .../distribution/particles.pxd                |  2 ++
 .../distribution/particles.pyx                | 20 ++++++++++++++++++-
 tests/test_particles.py                       | 16 +++++++--------
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/pomdp_py/representations/distribution/particles.pxd b/pomdp_py/representations/distribution/particles.pxd
index a8226be5..e2d0b3f5 100644
--- a/pomdp_py/representations/distribution/particles.pxd
+++ b/pomdp_py/representations/distribution/particles.pxd
@@ -8,6 +8,8 @@ cdef class WeightedParticles(GenerativeDistribution):
     cdef object _distance_func
     cdef dict _hist
     cdef bint _hist_valid
+    cdef bint _frozen
+    cdef int _hashcode
 
     cpdef dict get_histogram(self)
 
diff --git a/pomdp_py/representations/distribution/particles.pyx b/pomdp_py/representations/distribution/particles.pyx
index b462c50e..20ae6db1 100644
--- a/pomdp_py/representations/distribution/particles.pyx
+++ b/pomdp_py/representations/distribution/particles.pyx
@@ -1,4 +1,5 @@
 from pomdp_py.framework.basics cimport GenerativeDistribution
+from pomdp_py.utils.cython_utils cimport det_dict_hash
 import random
 
 cdef class WeightedParticles(GenerativeDistribution):
@@ -19,8 +20,10 @@ cdef class WeightedParticles(GenerativeDistribution):
             are comparable; "none" if no approximation, return 0.
        distance_func: Used when approx_method is 'nearest'. Returns
            a number given two values in this particle set.
+       frozen: if true, then this WeightedParticles object cannot be modified. This
+           makes it hashable.
     """
-    def __init__(self, list particles, str approx_method="none", distance_func=None):
+    def __init__(self, list particles, str approx_method="none", distance_func=None, frozen=False):
         self._values = [value for value, _ in particles]
         self._weights = [weight for _, weight in particles]
         self._particles = particles
@@ -30,6 +33,9 @@ cdef class WeightedParticles(GenerativeDistribution):
 
         self._approx_method = approx_method
         self._distance_func = distance_func
+        self._frozen = frozen
+        if self._frozen:
+            self._hashcode = det_dict_hash(self._hist)
 
     @property
     def particles(self):
@@ -46,6 +52,8 @@ cdef class WeightedParticles(GenerativeDistribution):
     def add(self, particle):
         """add(self, particle)
         particle: (value, weight) tuple"""
+        if self._frozen:
+            raise NotImplementedError("weighted particles is frozen and cannot be modified")
         self._particles.append(particle)
         s, w = particle
         self._values.append(s)
@@ -58,6 +66,16 @@ cdef class WeightedParticles(GenerativeDistribution):
     def __len__(self):
         return len(self._particles)
 
+    def __hash__(self):
+        if self._frozen:
+            return self._hashcode
+        raise NotImplementedError
+
+    def __eq__(self, other):
+        if isinstance(other, WeightedParticles):
+            return self._hist == other._hist
+        return False
+
     def __getitem__(self, value):
         """Returns the probability of `value`; normalized"""
         if len(self.particles) == 0:
diff --git a/tests/test_particles.py b/tests/test_particles.py
index 5470176c..6ffa4861 100644
--- a/tests/test_particles.py
+++ b/tests/test_particles.py
@@ -41,17 +41,16 @@ def test_weighted_particles():
     for v in range(5):
         random_dist[f"x{v}"] = random.uniform(0, 1)
         total_prob += random_dist[f"x{v}"]
-    for v in random_dist:
-        random_dist[v] /= total_prob
 
-    particles = pomdp_py.WeightedParticles.from_histogram(
-        pomdp_py.Histogram(random_dist)
-    )
+    particles = pomdp_py.WeightedParticles.from_histogram(random_dist)
+    particles_frozen = pomdp_py.WeightedParticles(particles.particles, frozen=True)
 
     assert abs(sum(particles[v] for v, _ in particles) - 1.0) <= 1e-6
+    assert abs(sum(particles_frozen[v] for v, _ in particles_frozen) - 1.0) <= 1e-6
 
     for v in random_dist:
-        assert abs(particles[v] - random_dist[v]) <= 2e-3
+        assert abs(particles[v] - random_dist[v] / total_prob) <= 2e-3
+        assert abs(particles_frozen[v] - random_dist[v] / total_prob) <= 2e-3
 
     counts = {}
     total = int(1e6)
@@ -61,10 +60,11 @@ def test_weighted_particles():
     for v in counts:
         counts[v] /= total
     for v in random_dist:
-        assert abs(counts[v] - random_dist[v]) <= 2e-3
+        assert abs(counts[v] - random_dist[v] / total_prob) <= 2e-3
 
     assert particles.mpe() == pomdp_py.Histogram(random_dist).mpe()
-
+    assert particles_frozen.mpe() == pomdp_py.Histogram(random_dist).mpe()
+    hash(particles_frozen)
 
 def run():
     test_particles()

From 18bf5880fcef2db80fbe26045ece458edc788b78 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 10:57:21 -0500
Subject: [PATCH 13/78] add the frozen property

---
 pomdp_py/representations/distribution/particles.pyx | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pomdp_py/representations/distribution/particles.pyx b/pomdp_py/representations/distribution/particles.pyx
index 20ae6db1..09b1b984 100644
--- a/pomdp_py/representations/distribution/particles.pyx
+++ b/pomdp_py/representations/distribution/particles.pyx
@@ -49,6 +49,10 @@ cdef class WeightedParticles(GenerativeDistribution):
     def weights(self):
         return self._weights
 
+    @property
+    def frozen(self):
+        return self._frozen
+
     def add(self, particle):
         """add(self, particle)
         particle: (value, weight) tuple"""

From 0d4ed8a2a0b2f53a1dec0055e24216dcd8172dec Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 12:41:03 -0500
Subject: [PATCH 14/78] remove spurious argument

---
 pomdp_py/algorithms/po_uct.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index 2b16bbcb..3fb191be 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -345,7 +345,7 @@ cdef class POUCT(Planner):
             return 0
         if root is None:
             if self._agent.tree is None:
-                root = self._VNode(agent=self._agent, root=True)
+                root = self._VNode(root=True)
                 self._agent.tree = root
                 if self._agent.tree.history != self._agent.history:
                     raise ValueError("Unable to plan for the given history.")
@@ -427,7 +427,7 @@ cdef class POUCT(Planner):
             reward = self._agent.reward_model.sample(state, action, next_state)
         return next_state, observation, reward
 
-    def _VNode(self, agent=None, root=False, **kwargs):
+    def _VNode(self, root=False, **kwargs):
         """Returns a VNode with default values; The function naming makes it clear
         that this function is about creating a VNode object."""
         if root:

From 5a2ee880c3500d31630c1609efdeefb476830963 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 12:56:36 -0500
Subject: [PATCH 15/78] refactor pouct _search to be easier for overriding

---
 pomdp_py/algorithms/po_uct.pxd |  6 +++
 pomdp_py/algorithms/po_uct.pyx | 73 +++++++++++++++-------------------
 2 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/pomdp_py/algorithms/po_uct.pxd b/pomdp_py/algorithms/po_uct.pxd
index 85ba55dd..9e28ff52 100644
--- a/pomdp_py/algorithms/po_uct.pxd
+++ b/pomdp_py/algorithms/po_uct.pxd
@@ -32,6 +32,12 @@ cdef class POUCT(Planner):
     cdef int _pbar_update_interval
 
     cpdef _search(self)
+    cdef _initialize_progress_bar(self)
+    cpdef _perform_simulation(self, state)
+    cdef bint _should_stop(self, int sims_count, float start_time)
+    cdef _update_progress(self, int sims_count, float start_time)
+    cdef _finalize_progress_bar(self)
+
     cpdef _simulate(POUCT self,
                     State state, tuple history, VNode root, QNode parent,
                     Observation observation, int depth)
diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index 3fb191be..8e5c5062 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -288,55 +288,46 @@ cdef class POUCT(Planner):
                                             value_init)
                 vnode[action] = history_action_node
 
-
     cpdef _search(self):
-        cdef State state
-        cdef Action best_action
         cdef int sims_count = 0
-        cdef float time_taken = 0
-        cdef float best_value
-        cdef bint stop_by_sims = self._num_sims > 0
-        cdef object pbar
+        cdef float start_time, time_taken
+        self._initialize_progress_bar()
+        start_time = time.time()
+
+        while not self._should_stop(sims_count, start_time):
+            state = self._agent.sample_belief()
+            self._perform_simulation(state)
+            sims_count += 1
+            self._update_progress(sims_count, start_time)
+
+        self._finalize_progress_bar()
+        best_action = self._agent.tree.argmax()
+        time_taken = time.time() - start_time
+        return best_action, time_taken, sims_count
 
+    cdef _initialize_progress_bar(self):
         if self._show_progress:
-            if stop_by_sims:
-                total = int(self._num_sims)
-            else:
-                total = self._planning_time
-            pbar = tqdm(total=total)
+            total = self._num_sims if self._num_sims > 0 else self._planning_time
+            self.pbar = tqdm(total=total)
 
-        start_time = time.time()
-        while True:
-            ## Note: the tree node with () history will have
-            ## the init belief given to the agent.
-            state = self._agent.sample_belief()
-            self._simulate(state, self._agent.history, self._agent.tree,
-                           None, None, 0)
-            sims_count +=1
-            time_taken = time.time() - start_time
-
-            if self._show_progress and sims_count % self._pbar_update_interval == 0:
-                if stop_by_sims:
-                    pbar.n = sims_count
-                else:
-                    pbar.n = time_taken
-                pbar.refresh()
-
-            if stop_by_sims:
-                if sims_count >= self._num_sims:
-                    break
-            else:
-                if time_taken > self._planning_time:
-                    if self._show_progress:
-                        pbar.n = self._planning_time
-                        pbar.refresh()
-                    break
+    cpdef _perform_simulation(self, state):
+        self._simulate(state, self._agent.history, self._agent.tree, None, None, 0)
 
+    cdef bint _should_stop(self, int sims_count, float start_time):
+        cdef float time_taken = time.time() - start_time
+        if self._num_sims > 0:
+            return sims_count >= self._num_sims
+        else:
+            return time_taken > self._planning_time
+
+    cdef _update_progress(self, int sims_count, float start_time):
         if self._show_progress:
-            pbar.close()
+            self.pbar.n = sims_count if self._num_sims > 0 else time.time() - start_time
+            self.pbar.refresh()
 
-        best_action = self._agent.tree.argmax()
-        return best_action, time_taken, sims_count
+    cdef _finalize_progress_bar(self):
+        if self._show_progress:
+            self.pbar.close()
 
     cpdef _simulate(POUCT self,
                     State state, tuple history, VNode root, QNode parent,

From ca9224c9e8bb34d8544b2594b3faaf2f0dc75575 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 12:58:32 -0500
Subject: [PATCH 16/78] black

---
 tests/test_particles.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_particles.py b/tests/test_particles.py
index 6ffa4861..bb0a5f80 100644
--- a/tests/test_particles.py
+++ b/tests/test_particles.py
@@ -66,6 +66,7 @@ def test_weighted_particles():
     assert particles_frozen.mpe() == pomdp_py.Histogram(random_dist).mpe()
     hash(particles_frozen)
 
+
 def run():
     test_particles()
     test_weighted_particles()

From e0f9b686b51966291735e6f3507348e7366a3e55 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 13:18:30 -0500
Subject: [PATCH 17/78] comment correction

---
 pomdp_py/framework/basics.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pomdp_py/framework/basics.pyx b/pomdp_py/framework/basics.pyx
index 0640c16d..d53c0b35 100644
--- a/pomdp_py/framework/basics.pyx
+++ b/pomdp_py/framework/basics.pyx
@@ -194,7 +194,7 @@ cdef class BlackboxModel:
     def sample(self, state, action):
         """
         sample(self, state, action)
-        Sample (s',o,r) ~ G(s',o,r)"""
+        Sample (s',o,r) ~ G(s,a)"""
         raise NotImplementedError
 
     def argmax(self, state, action):

From 0afa801fb9b3d2727e45c679a8a4431c97582efb Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 14:37:17 -0500
Subject: [PATCH 18/78] type fix. time.time returns a double

---
 pomdp_py/algorithms/po_uct.pxd | 4 ++--
 pomdp_py/algorithms/po_uct.pyx | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pomdp_py/algorithms/po_uct.pxd b/pomdp_py/algorithms/po_uct.pxd
index 9e28ff52..2e1b8783 100644
--- a/pomdp_py/algorithms/po_uct.pxd
+++ b/pomdp_py/algorithms/po_uct.pxd
@@ -34,8 +34,8 @@ cdef class POUCT(Planner):
     cpdef _search(self)
     cdef _initialize_progress_bar(self)
     cpdef _perform_simulation(self, state)
-    cdef bint _should_stop(self, int sims_count, float start_time)
-    cdef _update_progress(self, int sims_count, float start_time)
+    cdef bint _should_stop(self, int sims_count, double start_time)
+    cdef _update_progress(self, int sims_count, double start_time)
     cdef _finalize_progress_bar(self)
 
     cpdef _simulate(POUCT self,
diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index 8e5c5062..31daa151 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -290,7 +290,7 @@ cdef class POUCT(Planner):
 
     cpdef _search(self):
         cdef int sims_count = 0
-        cdef float start_time, time_taken
+        cdef double start_time, time_taken
         self._initialize_progress_bar()
         start_time = time.time()
 
@@ -313,14 +313,14 @@ cdef class POUCT(Planner):
     cpdef _perform_simulation(self, state):
         self._simulate(state, self._agent.history, self._agent.tree, None, None, 0)
 
-    cdef bint _should_stop(self, int sims_count, float start_time):
+    cdef bint _should_stop(self, int sims_count, double start_time):
         cdef float time_taken = time.time() - start_time
         if self._num_sims > 0:
             return sims_count >= self._num_sims
         else:
             return time_taken > self._planning_time
 
-    cdef _update_progress(self, int sims_count, float start_time):
+    cdef _update_progress(self, int sims_count, double start_time):
         if self._show_progress:
             self.pbar.n = sims_count if self._num_sims > 0 else time.time() - start_time
             self.pbar.refresh()

From 1e3e5888a092d8cd5f4909bafa55c87827910db4 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 14:37:32 -0500
Subject: [PATCH 19/78] remove spurious argument for pomcp

---
 pomdp_py/algorithms/pomcp.pyx | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/pomdp_py/algorithms/pomcp.pyx b/pomdp_py/algorithms/pomcp.pyx
index 47b5ffa6..349b8127 100644
--- a/pomdp_py/algorithms/pomcp.pyx
+++ b/pomdp_py/algorithms/pomcp.pyx
@@ -133,18 +133,14 @@ cdef class POMCP(POUCT):
             root.belief.add(state)  # belief update happens as simulation goes.
         return total_reward
 
-    def _VNode(self, agent=None, root=False, **kwargs):
+    def _VNode(self, root=False, **kwargs):
         """Returns a VNode with default values; The function naming makes it clear
         that this function is about creating a VNode object."""
         if root:
             # agent cannot be None.
             return RootVNodeParticles(self._num_visits_init,
-                                      agent.history,
-                                      belief=copy.deepcopy(agent.belief))
+                                      self._agent.history,
+                                      belief=copy.deepcopy(self._agent.belief))
         else:
-            if agent is None:
-                return VNodeParticles(self._num_visits_init,
-                                      belief=Particles([]))
-            else:
-                return VNodeParticles(self._num_visits_init,
-                                      belief=copy.deepcopy(agent.belief))
+            return VNodeParticles(self._num_visits_init,
+                                  belief=Particles([]))

From 538c5f9d69cd9ec627f7cfa1cac7da0182a7abb7 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 17:20:28 -0500
Subject: [PATCH 20/78] additional attributes

---
 pomdp_py/algorithms/po_uct.pyx | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index 31daa151..686c142d 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -222,6 +222,22 @@ cdef class POUCT(Planner):
         """Returns the amount of time (seconds) ran for the last `plan` call."""
         return self._last_planning_time
 
+    @property
+    def max_depth(self):
+        return self._max_depth
+
+    @property
+    def num_visits_init(self):
+        return self._num_visits_init
+
+    @property
+    def value_init(self):
+        return self._value_init
+
+    @property
+    def action_prior(self):
+        return self._action_prior
+
     cpdef public plan(self, Agent agent):
         cdef Action action
         cdef float time_taken

From d7463827024a7221ab424be2049fe45b819e6297 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 17:20:34 -0500
Subject: [PATCH 21/78] det_dict_hash without sorting

---
 pomdp_py/utils/cython_utils.pyx | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/pomdp_py/utils/cython_utils.pyx b/pomdp_py/utils/cython_utils.pyx
index 613ff4f3..ba326254 100644
--- a/pomdp_py/utils/cython_utils.pyx
+++ b/pomdp_py/utils/cython_utils.pyx
@@ -3,7 +3,13 @@
 import hashlib
 
 cpdef det_dict_hash(dct, keep=9):
-    """deterministic hash of a dictionary."""
-    content = str(list(sorted(dct.items()))).encode()
-    hashcode = int(str(int(hashlib.sha1(content).hexdigest(), 16))[:keep])
+    """Deterministic hash of a dictionary without sorting."""
+    hash_accumulator = 0
+    for key, value in dct.items():
+        pair_str = f"{key}:{value}".encode()
+        pair_hash = hashlib.sha1(pair_str).hexdigest()
+        hash_accumulator += int(pair_hash, 16)
+
+    # Convert the accumulated hash back to a string, take the first 'keep' digits, and convert to an integer
+    hashcode = int(str(hash_accumulator)[:keep])
     return hashcode

From c7d23c94124bcb1f4cad2a86d7f1ce2f1d9dae87 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 17:20:47 -0500
Subject: [PATCH 22/78] add frozen

---
 pomdp_py/representations/distribution/particles.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pomdp_py/representations/distribution/particles.pyx b/pomdp_py/representations/distribution/particles.pyx
index 09b1b984..7997dd94 100644
--- a/pomdp_py/representations/distribution/particles.pyx
+++ b/pomdp_py/representations/distribution/particles.pyx
@@ -148,13 +148,13 @@ cdef class WeightedParticles(GenerativeDistribution):
         return hist
 
     @classmethod
-    def from_histogram(cls, histogram):
+    def from_histogram(cls, histogram, frozen=False):
         """Given a pomdp_py.Histogram return a particle representation of it,
         which is an approximation"""
         particles = []
         for v in histogram:
             particles.append((v, histogram[v]))
-        return WeightedParticles(particles)
+        return WeightedParticles(particles, frozen=frozen)
 
     def condense(self):
         """

From 08495a137a3a5ec9e6be0ece22c3e2202c55ba11 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 17:22:33 -0500
Subject: [PATCH 23/78] one more attr

---
 pomdp_py/algorithms/po_uct.pyx | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index 686c142d..6e5f49e8 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -238,6 +238,10 @@ cdef class POUCT(Planner):
     def action_prior(self):
         return self._action_prior
 
+    @property
+    def rollout_policy(self):
+        return self._rollout_policy
+
     cpdef public plan(self, Agent agent):
         cdef Action action
         cdef float time_taken

From 512b192999d28d543fa1d08fb470d80e195d76e0 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 17:31:56 -0500
Subject: [PATCH 24/78] rollout policy should default to none

---
 pomdp_py/algorithms/po_uct.pyx | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index 6e5f49e8..d17e78e3 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -186,7 +186,7 @@ cdef class POUCT(Planner):
                  max_depth=5, planning_time=-1., num_sims=-1,
                  discount_factor=0.9, exploration_const=math.sqrt(2),
                  num_visits_init=0, value_init=0,
-                 rollout_policy=RandomRollout(),
+                 rollout_policy=None,
                  action_prior=None, show_progress=False, pbar_update_interval=5):
         self._max_depth = max_depth
         self._planning_time = planning_time
@@ -247,6 +247,10 @@ cdef class POUCT(Planner):
         cdef float time_taken
         cdef int sims_count
 
+        if self._rollout_policy is None:
+            raise ValueError("rollout_policy unset. Please call set_rollout_policy, "
+                             "or pass in a rollout_policy upon initialization")
+
         self._agent = agent   # switch focus on planning for the given agent
         if not hasattr(self._agent, "tree"):
             self._agent.add_attr("tree", None)

From 9bab4e960d40b4ddac0bd8f419b37accf5903abf Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 4 Mar 2024 18:53:48 -0500
Subject: [PATCH 25/78] one more property

---
 pomdp_py/algorithms/po_uct.pyx | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index d17e78e3..a317d92b 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -230,6 +230,10 @@ cdef class POUCT(Planner):
     def num_visits_init(self):
         return self._num_visits_init
 
+    @property
+    def discount_factor(self):
+        return self._discount_factor
+
     @property
     def value_init(self):
         return self._value_init

From 4897adfe36b7094b3979833dd68e25f880f688c7 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 10:42:18 -0500
Subject: [PATCH 26/78] pbar

---
 pomdp_py/algorithms/po_uct.pxd |  4 ++--
 pomdp_py/algorithms/po_uct.pyx | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/pomdp_py/algorithms/po_uct.pxd b/pomdp_py/algorithms/po_uct.pxd
index 2e1b8783..6f66fffd 100644
--- a/pomdp_py/algorithms/po_uct.pxd
+++ b/pomdp_py/algorithms/po_uct.pxd
@@ -35,8 +35,8 @@ cdef class POUCT(Planner):
     cdef _initialize_progress_bar(self)
     cpdef _perform_simulation(self, state)
     cdef bint _should_stop(self, int sims_count, double start_time)
-    cdef _update_progress(self, int sims_count, double start_time)
-    cdef _finalize_progress_bar(self)
+    cdef _update_progress(self, pbar, int sims_count, double start_time)
+    cdef _finalize_progress_bar(self, pbar)
 
     cpdef _simulate(POUCT self,
                     State state, tuple history, VNode root, QNode parent,
diff --git a/pomdp_py/algorithms/po_uct.pyx b/pomdp_py/algorithms/po_uct.pyx
index a317d92b..c0f02665 100644
--- a/pomdp_py/algorithms/po_uct.pyx
+++ b/pomdp_py/algorithms/po_uct.pyx
@@ -319,16 +319,16 @@ cdef class POUCT(Planner):
     cpdef _search(self):
         cdef int sims_count = 0
         cdef double start_time, time_taken
-        self._initialize_progress_bar()
+        pbar = self._initialize_progress_bar()
         start_time = time.time()
 
         while not self._should_stop(sims_count, start_time):
             state = self._agent.sample_belief()
             self._perform_simulation(state)
             sims_count += 1
-            self._update_progress(sims_count, start_time)
+            self._update_progress(pbar, sims_count, start_time)
 
-        self._finalize_progress_bar()
+        self._finalize_progress_bar(pbar)
         best_action = self._agent.tree.argmax()
         time_taken = time.time() - start_time
         return best_action, time_taken, sims_count
@@ -336,7 +336,7 @@ cdef class POUCT(Planner):
     cdef _initialize_progress_bar(self):
         if self._show_progress:
             total = self._num_sims if self._num_sims > 0 else self._planning_time
-            self.pbar = tqdm(total=total)
+            return tqdm(total=total)
 
     cpdef _perform_simulation(self, state):
         self._simulate(state, self._agent.history, self._agent.tree, None, None, 0)
@@ -348,14 +348,14 @@ cdef class POUCT(Planner):
         else:
             return time_taken > self._planning_time
 
-    cdef _update_progress(self, int sims_count, double start_time):
+    cdef _update_progress(self, pbar, int sims_count, double start_time):
         if self._show_progress:
-            self.pbar.n = sims_count if self._num_sims > 0 else time.time() - start_time
-            self.pbar.refresh()
+            pbar.n = sims_count if self._num_sims > 0 else round(time.time() - start_time, 2)
+            pbar.refresh()
 
-    cdef _finalize_progress_bar(self):
+    cdef _finalize_progress_bar(self, pbar):
         if self._show_progress:
-            self.pbar.close()
+            pbar.close()
 
     cpdef _simulate(POUCT self,
                     State state, tuple history, VNode root, QNode parent,

From 03f27641b31dd59934b45ee9aaf9fa20e16381d8 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 10:47:27 -0500
Subject: [PATCH 27/78] adding release script

---
 scripts/README.md           |  33 ++++++++++
 scripts/pomdp_py_release.sh | 122 ++++++++++++++++++++++++++++++++++++
 2 files changed, 155 insertions(+)
 create mode 100644 scripts/README.md
 create mode 100755 scripts/pomdp_py_release.sh

diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 00000000..e4c5169f
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,33 @@
+# pomdp-py-release
+Convenience for releasing pomdp_py
+
+
+How to release pomdp-py:
+
+1. For each python version you'd like to support, starting from low to high,
+2. activate the virtulenv for that python version
+3. Run `./release.sh <version>`
+
+   (note that if pomdp-py repo is assumed to be at `$HOME/repo/pomdp-py`.
+
+   This should generate both wheel and source builds,
+   and place them under `pomdp-py/dist` properly.
+
+4. After you've generated the wheels for all the versions of your interest,
+   push to PyPI using twine:
+
+    ```
+    python3 -m twine upload --repository pypi dist/*
+    ```
+
+     ![image](https://github.com/zkytony/pomdp-py-release/assets/7720184/16f272bf-0996-464a-8678-34f3a150b890)
+
+
+## Repository Convention (est. 07/25/23)
+
+* When you're executing a release, besides the steps above, you should always:
+
+   1. Merge the technical content of the `dev-<version>` branch into `master`.
+   2. Create a separate PR with all the docs updates.
+
+ This makes it so that the contributions of external collaborators are correctly counted by github. Otherwise, the Sphinx html doc builds will count thousands of lines of changes to the collaborators which are incorrect.
diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
new file mode 100755
index 00000000..3742075a
--- /dev/null
+++ b/scripts/pomdp_py_release.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+# Example usage:
+#
+#   ./release.sh 1.3.3
+
+# Define the function.
+find_pxd_files_and_write_manifest() {
+    dir=$1
+    output_file=$2
+    find "$dir" -name '*.pxd'| while read -r line; do
+        echo "include $line"
+    done > "$output_file"
+    find "$dir" -name '*.pyx'| while read -r line; do
+        echo "include $line"
+    done >> "$output_file"
+}
+
+# Define the function.
+is_git_repo_on_branch() {
+    repo_path=$1
+    branch_name=$2
+
+    # Move to the git repository
+    cd "$repo_path" || return 1
+
+    # Get the current branch name
+    current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+    # Check if the current branch is the expected one
+    if [ "$current_branch" = "$branch_name" ]; then
+        true && return
+    else
+        false
+    fi
+}
+
+does_docker_image_exist() {
+    image_name=$1
+
+    # Check if the Docker image exists locally
+    if [[ "$(docker images -q "$image_name" 2> /dev/null)" == "" ]]; then
+        # If the image doesn't exist locally, pull it
+        false
+    else
+        true && return
+    fi
+}
+
+get_python_version() {
+    version_string=$(python3 --version 2>&1)
+    version=$(echo "$version_string" | awk '{print $2}')
+    major_version=${version%%.*}
+    minor_version=$(echo "$version" | cut -d. -f2)
+    echo "cp${major_version}${minor_version}"
+}
+
+user_pwd=$PWD
+
+# Write the MANIFEST.in file
+pomdp_py_path=$HOME/repo/pomdp-py
+cd $pomdp_py_path
+find_pxd_files_and_write_manifest ./ MANIFEST.in
+
+# Check if pomdp-py is on the right branch
+if [ $# -ne 1 ]; then
+    echo -e "Usage: $0 <version>\n"
+    echo -e "Help:"
+    echo -e "   version: version of pomdp-py release, e.g. 1.3.3"
+    exit 1
+fi
+
+version=$1
+if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
+   echo "pomdp-py repo is not on the branch dev-$version that you want release for. Abort."
+   exit 1
+fi
+echo -e "========= making release for pomdp-py $version ========="
+
+pip install setuptools
+pip install Cython
+
+# Note that we are building with pyproject.toml
+python3 setup.py build_ext --inplace
+pip install build
+python -m build
+
+# create the manylinux container
+linux_dist=manylinux2014_x86_64
+manylinux_image=quay.io/pypa/$linux_dist
+if ! does_docker_image_exist $manylinux_image; then
+   docker pull $manylinux_image
+fi
+cpv=$(get_python_version)
+wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
+command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
+docker run -it --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image  bash -c "$command"
+rm $pomdp_py_path/dist/$wheel_name
+fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
+sudo chown -R $(whoami).$(whoami) "$pomdp_py_path/wheelhouse/"
+mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
+rm -r $pomdp_py_path/wheelhouse
+
+# Verification (wheel)
+echo -e "------------ verification: wheel ---------"
+pip uninstall pomdp_py
+pip install "$pomdp_py_path/dist/$fixed_wheel_name"
+python $pomdp_py_path/tests/test_all.py
+
+# Verification (source)
+echo -e "------------ verification: source ---------"
+pip uninstall pomdp_py
+cd $pomdp_py_path/dist
+pip install pomdp-py-$version.tar.gz
+python $pomdp_py_path/tests/test_all.py
+
+pip install twine
+echo -e "If successful, run"
+echo -e "    python3 -m twine upload --repository pypi $pomdp_py_path/dist/*"
+echo -e "to upload the release to PyPI."
+
+
+cd $user_pwd

From 6f6665172569cec6e0559e5274f72194550455cb Mon Sep 17 00:00:00 2001
From: kzheng <125413689+kaiyu-zheng@users.noreply.github.com>
Date: Tue, 5 Mar 2024 10:50:01 -0500
Subject: [PATCH 28/78] Update scripts/README.md - remove image

---
 scripts/README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/README.md b/scripts/README.md
index e4c5169f..3ef5c776 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -20,8 +20,6 @@ How to release pomdp-py:
     python3 -m twine upload --repository pypi dist/*
     ```
 
-     ![image](https://github.com/zkytony/pomdp-py-release/assets/7720184/16f272bf-0996-464a-8678-34f3a150b890)
-
 
 ## Repository Convention (est. 07/25/23)
 

From 7cb9745bf3ab40cec5447571e64d9ca34f3e2526 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 11:48:59 -0500
Subject: [PATCH 29/78] include pomdp-solve in manifest

---
 scripts/pomdp_py_release.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 3742075a..2680e857 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -13,6 +13,8 @@ find_pxd_files_and_write_manifest() {
     find "$dir" -name '*.pyx'| while read -r line; do
         echo "include $line"
     done >> "$output_file"
+    # Include pomdp-solve as well
+    echo "include ./thirdparty/pomdp-solve/src/pomdp-solve" >> "$output_file"
 }
 
 # Define the function.

From 928b3c0ed8aa822c360d49dbdd6da3a26797559f Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 11:49:16 -0500
Subject: [PATCH 30/78] properly deal with build version while on dev-latest

---
 scripts/pomdp_py_release.sh | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 2680e857..c37dca63 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -56,10 +56,18 @@ get_python_version() {
     echo "cp${major_version}${minor_version}"
 }
 
+extract_package_version() {
+    local pyproject_path="$1"
+    local version_line=$(grep '^version\s*=\s*"' "$pyproject_path" | head -n 1)
+    local version=$(echo "$version_line" | sed -E 's/version\s*=\s*"([^"]+)"/\1/')
+    echo "$version"
+}
+
 user_pwd=$PWD
 
 # Write the MANIFEST.in file
-pomdp_py_path=$HOME/repo/pomdp-py
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+pomdp_py_path=$SCRIPTPATH/../
 cd $pomdp_py_path
 find_pxd_files_and_write_manifest ./ MANIFEST.in
 
@@ -71,11 +79,22 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
-version=$1
+version=$(extract_package_version "$pomdp_py_path/pyproject.toml")
 if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
-   echo "pomdp-py repo is not on the branch dev-$version that you want release for. Abort."
-   exit 1
+    if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then
+        echo "pomdp-py repo must be either on dev-latest or dev-$version. Abort"
+        exit 1
+    fi
 fi
+
+echo -e "========= build pomdp-solve ========="
+if [ ! -f $pomdp_py_path/thirdparty/pomdp-solve/src/pomdp-solve ]; then
+    cd $pomdp_py_path/thirdparty/pomdp-solve
+    ./configure
+    make
+    cd $pomdp_py_path
+fi
+
 echo -e "========= making release for pomdp-py $version ========="
 
 pip install setuptools

From 2342920366bc91e858fa280b26bc5b9e159710f6 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 11:59:17 -0500
Subject: [PATCH 31/78] script doesn't need an argument

---
 scripts/pomdp_py_release.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index c37dca63..876c4e30 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # Example usage:
 #
-#   ./release.sh 1.3.3
+#   ./release.sh
 
 # Define the function.
 find_pxd_files_and_write_manifest() {

From 2ec0be628baccc90c3f40483ec1aaffd275464e1 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 11:59:46 -0500
Subject: [PATCH 32/78] include pomdp-solve binary when building pomdp-py

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 70655dda..8ef8429f 100644
--- a/setup.py
+++ b/setup.py
@@ -41,10 +41,10 @@ def build_extensions(pkg_name, major_submodules):
     ext_modules=cythonize(
         extensions, build_dir="build", compiler_directives={"language_level": "3"}
     ),
-    packages=find_packages(exclude=["thirdparty", "thirdparty.*"]),
+    packages=find_packages(),
     package_data={
-        "pomdp_py": ["*.pxd", "*.pyx", "*.so", "*.c"],
-        "pomdp_problems": ["*.pxd", "*.pyx", "*.so", "*.c"],
+        "pomdp_py": ["*.pxd", "*.pyx", "*.so", "*.c",
+                     "thirdparty/pomdp-solve/src/pomdp-solve"],
     },
     zip_safe=False,
 )

From cd598585b302765c53b6602ae8c3773726807964 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 12:00:17 -0500
Subject: [PATCH 33/78] version should match convention

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 2db1ad63..ff42c20b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pomdp-py"
-version = "1.3.5-dev"
+version = "1.3.5"
 description = "Python POMDP Library."
 authors = [{name = "Kaiyu Zheng", email = "kzheng10@cs.brown.edu"}]
 license = {file = "LICENSE"}

From 828be8973051d85452e790080e58ef428e82286a Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 12:00:26 -0500
Subject: [PATCH 34/78] regenerate manifest

---
 MANIFEST.in | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index f0f5e423..c9d53d8c 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,26 +1,27 @@
-include ./pomdp_py/utils/cython_utils.pxd
-include ./pomdp_py/algorithms/po_uct.pxd
-include ./pomdp_py/algorithms/po_rollout.pxd
-include ./pomdp_py/algorithms/pomcp.pxd
-include ./pomdp_py/algorithms/value_iteration.pxd
-include ./pomdp_py/representations/belief/particles.pxd
 include ./pomdp_py/representations/distribution/particles.pxd
-include ./pomdp_py/representations/distribution/gaussian.pxd
 include ./pomdp_py/representations/distribution/histogram.pxd
+include ./pomdp_py/representations/distribution/gaussian.pxd
+include ./pomdp_py/representations/belief/particles.pxd
+include ./pomdp_py/utils/cython_utils.pxd
+include ./pomdp_py/framework/oopomdp.pxd
 include ./pomdp_py/framework/planner.pxd
 include ./pomdp_py/framework/basics.pxd
-include ./pomdp_py/framework/oopomdp.pxd
-include ./pomdp_py/utils/cython_utils.pyx
+include ./pomdp_py/algorithms/po_rollout.pxd
+include ./pomdp_py/algorithms/pomcp.pxd
+include ./pomdp_py/algorithms/po_uct.pxd
+include ./pomdp_py/algorithms/value_iteration.pxd
 include ./pomdp_py/problems/rocksample/cythonize/rocksample_problem.pyx
 include ./pomdp_py/problems/tiger/cythonize/tiger_problem.pyx
-include ./pomdp_py/algorithms/value_iteration.pyx
-include ./pomdp_py/algorithms/pomcp.pyx
-include ./pomdp_py/algorithms/po_rollout.pyx
-include ./pomdp_py/algorithms/po_uct.pyx
-include ./pomdp_py/representations/belief/particles.pyx
-include ./pomdp_py/representations/distribution/particles.pyx
 include ./pomdp_py/representations/distribution/histogram.pyx
 include ./pomdp_py/representations/distribution/gaussian.pyx
-include ./pomdp_py/framework/basics.pyx
+include ./pomdp_py/representations/distribution/particles.pyx
+include ./pomdp_py/representations/belief/particles.pyx
+include ./pomdp_py/utils/cython_utils.pyx
 include ./pomdp_py/framework/planner.pyx
+include ./pomdp_py/framework/basics.pyx
 include ./pomdp_py/framework/oopomdp.pyx
+include ./pomdp_py/algorithms/po_rollout.pyx
+include ./pomdp_py/algorithms/value_iteration.pyx
+include ./pomdp_py/algorithms/pomcp.pyx
+include ./pomdp_py/algorithms/po_uct.pyx
+include ./thirdparty/pomdp-solve/src/pomdp-solve

From 9b6825e9ccc0f38792dafed75fea5e460d7bce8c Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 13:12:03 -0500
Subject: [PATCH 35/78] do not build thirdparty binary into wheel

---
 scripts/pomdp_py_release.sh | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 876c4e30..4953b799 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -1,4 +1,11 @@
 #!/bin/bash
+#
+# Creates a wheel for pomdp-py and prompts user how to release it on
+# pypi.  Note that the wheel does not contain thirdparty binaries such
+# as pomdp-solve because we cannot ensure compatibility of the
+# pre-built binary. If needed, the user can clone the pomdp-py repo
+# and build those binary from within the thirdparty folder.
+#
 # Example usage:
 #
 #   ./release.sh
@@ -87,14 +94,6 @@ if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
     fi
 fi
 
-echo -e "========= build pomdp-solve ========="
-if [ ! -f $pomdp_py_path/thirdparty/pomdp-solve/src/pomdp-solve ]; then
-    cd $pomdp_py_path/thirdparty/pomdp-solve
-    ./configure
-    make
-    cd $pomdp_py_path
-fi
-
 echo -e "========= making release for pomdp-py $version ========="
 
 pip install setuptools

From d26895a13c6a56a8403430a9410dd3049a9f23cf Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 13:12:34 -0500
Subject: [PATCH 36/78] restore setup.py

---
 setup.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 8ef8429f..c519ea91 100644
--- a/setup.py
+++ b/setup.py
@@ -41,10 +41,9 @@ def build_extensions(pkg_name, major_submodules):
     ext_modules=cythonize(
         extensions, build_dir="build", compiler_directives={"language_level": "3"}
     ),
-    packages=find_packages(),
+    packages=find_packages(exclude=["thirdparty", "thirdparty.*"]),
     package_data={
-        "pomdp_py": ["*.pxd", "*.pyx", "*.so", "*.c",
-                     "thirdparty/pomdp-solve/src/pomdp-solve"],
+        "pomdp_py": ["*.pxd", "*.pyx", "*.so", "*.c"],
     },
     zip_safe=False,
 )

From 58b1cc6a4d9e9660dc72232e1278307f0cddb80b Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 13:46:42 -0500
Subject: [PATCH 37/78] build wheel workflow

---
 .github/workflows/build_wheel.yml | 36 +++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 .github/workflows/build_wheel.yml

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
new file mode 100644
index 00000000..abd4d34e
--- /dev/null
+++ b/.github/workflows/build_wheel.yml
@@ -0,0 +1,36 @@
+name: pomdp-py build dev-latest wheel
+
+on:
+  push:
+    branches: [ master, dev-latest ]
+  pull_request:
+    branches: [ master, dev-latest ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+    steps:
+    - uses: actions/checkout@v4
+      ref: 'dev-latest'
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+    - name: Run release script
+      working-directory: .
+      run: |
+        cd scripts
+        ./pomdp_py_release.sh
+    - name: Upload artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: pomdp-py wheel
+        path: dist/*.whl

From 190a8d6cbe015204bc508a938fce8969b0a108a4 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 14:05:13 -0500
Subject: [PATCH 38/78] typo fix

---
 .github/workflows/build_wheel.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
index abd4d34e..ca757ab3 100644
--- a/.github/workflows/build_wheel.yml
+++ b/.github/workflows/build_wheel.yml
@@ -17,7 +17,7 @@ jobs:
       fail-fast: true
     steps:
     - uses: actions/checkout@v4
-      ref: 'dev-latest'
+      refs: 'dev-latest'
     - uses: actions/setup-python@v5
       with:
         python-version: '3.10'

From 7e2b5f0b4bd63ba78e70c3e429346d5c4b67f466 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 14:06:13 -0500
Subject: [PATCH 39/78] workflow fix

---
 .github/workflows/build_wheel.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
index ca757ab3..2a831339 100644
--- a/.github/workflows/build_wheel.yml
+++ b/.github/workflows/build_wheel.yml
@@ -17,7 +17,8 @@ jobs:
       fail-fast: true
     steps:
     - uses: actions/checkout@v4
-      refs: 'dev-latest'
+      with:
+        refs: 'dev-latest'
     - uses: actions/setup-python@v5
       with:
         python-version: '3.10'

From 8ec2eea9120507ba13d6a5523a58ccdfb0409b8c Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 14:07:44 -0500
Subject: [PATCH 40/78] release script no arg

---
 scripts/pomdp_py_release.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 4953b799..d82a7556 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -79,13 +79,6 @@ cd $pomdp_py_path
 find_pxd_files_and_write_manifest ./ MANIFEST.in
 
 # Check if pomdp-py is on the right branch
-if [ $# -ne 1 ]; then
-    echo -e "Usage: $0 <version>\n"
-    echo -e "Help:"
-    echo -e "   version: version of pomdp-py release, e.g. 1.3.3"
-    exit 1
-fi
-
 version=$(extract_package_version "$pomdp_py_path/pyproject.toml")
 if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
     if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then

From ec06d97b2c4bcc28d6eb17b310d6a85120fe04fb Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 14:11:38 -0500
Subject: [PATCH 41/78] print current branch for debugging

---
 scripts/pomdp_py_release.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index d82a7556..dbe17129 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -82,7 +82,7 @@ find_pxd_files_and_write_manifest ./ MANIFEST.in
 version=$(extract_package_version "$pomdp_py_path/pyproject.toml")
 if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
     if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then
-        echo "pomdp-py repo must be either on dev-latest or dev-$version. Abort"
+        echo "pomdp-py repo must be either on dev-latest or dev-$version, but not $current_branch. Abort"
         exit 1
     fi
 fi

From ccae3672a7e2923ae69493ce962721f6dd2bc668 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 14:14:05 -0500
Subject: [PATCH 42/78] fix typo

---
 .github/workflows/build_wheel.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
index 2a831339..bc62a6b7 100644
--- a/.github/workflows/build_wheel.yml
+++ b/.github/workflows/build_wheel.yml
@@ -18,7 +18,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
       with:
-        refs: 'dev-latest'
+        ref: 'dev-latest'
     - uses: actions/setup-python@v5
       with:
         python-version: '3.10'

From ea56e8cb5fb28d0d34611ee3635ffb634c9614b9 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 14:44:52 -0500
Subject: [PATCH 43/78] non interactive and non sudo for github action friendly

---
 scripts/pomdp_py_release.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index dbe17129..293d4620 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -106,10 +106,10 @@ fi
 cpv=$(get_python_version)
 wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
 command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
-docker run -it --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image  bash -c "$command"
+docker run --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image  bash -c "$command"
 rm $pomdp_py_path/dist/$wheel_name
 fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
-sudo chown -R $(whoami).$(whoami) "$pomdp_py_path/wheelhouse/"
+chown -R $(whoami).$(whoami) "$pomdp_py_path/wheelhouse/"
 mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
 rm -r $pomdp_py_path/wheelhouse
 

From caa7746729030489d356dad2ab15e878d2257687 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 5 Mar 2024 14:56:30 -0500
Subject: [PATCH 44/78] no sudo chown

---
 scripts/pomdp_py_release.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 293d4620..c192f4df 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -106,10 +106,9 @@ fi
 cpv=$(get_python_version)
 wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
 command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
-docker run --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image  bash -c "$command"
+docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
 rm $pomdp_py_path/dist/$wheel_name
 fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
-chown -R $(whoami).$(whoami) "$pomdp_py_path/wheelhouse/"
 mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
 rm -r $pomdp_py_path/wheelhouse
 

From 755a9998da6c8238ba5cb94ffeca5339f5613844 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Wed, 6 Mar 2024 09:31:21 -0500
Subject: [PATCH 45/78] testing wheel release with action-gh-release

---
 .github/workflows/build_wheel.yml | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
index bc62a6b7..397b78fe 100644
--- a/.github/workflows/build_wheel.yml
+++ b/.github/workflows/build_wheel.yml
@@ -2,9 +2,7 @@ name: pomdp-py build dev-latest wheel
 
 on:
   push:
-    branches: [ master, dev-latest ]
-  pull_request:
-    branches: [ master, dev-latest ]
+    branches: [ dev-latest ]  # trigger on pushes to the dev-latest branch
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -25,13 +23,17 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-    - name: Run release script
-      working-directory: .
+    # - name: Run release script
+    #   working-directory: .
+    #   run: |
+    #     cd scripts
+    #     ./pomdp_py_release.sh
+    - name: Fake artifact
       run: |
-        cd scripts
-        ./pomdp_py_release.sh
-    - name: Upload artifact
-      uses: actions/upload-artifact@v4
+        mkdir dist
+        echo "HELLO" >> test.whl
+    - name: Release artifact
+      uses: softprops/action-gh-release@v1
       with:
         name: pomdp-py wheel
-        path: dist/*.whl
+        files: dist/*.whl

From 87246a085687982faff12bbeafcb7c5c1e7fe4d4 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Wed, 6 Mar 2024 09:40:59 -0500
Subject: [PATCH 46/78] trigger for wheel build should be tag

---
 .github/workflows/build_wheel.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
index 397b78fe..4037360c 100644
--- a/.github/workflows/build_wheel.yml
+++ b/.github/workflows/build_wheel.yml
@@ -2,7 +2,8 @@ name: pomdp-py build dev-latest wheel
 
 on:
   push:
-    branches: [ dev-latest ]  # trigger on pushes to the dev-latest branch
+    tags:
+      - 'vdev-latest'  # trigger on the tag vdev-latest
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

From 22a974aade2817f59b8770246b2f809de956f571 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Wed, 6 Mar 2024 12:04:16 -0500
Subject: [PATCH 47/78] release name in workflow

---
 .github/workflows/build_wheel.yml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
index 4037360c..c973964f 100644
--- a/.github/workflows/build_wheel.yml
+++ b/.github/workflows/build_wheel.yml
@@ -15,12 +15,19 @@ jobs:
     strategy:
       fail-fast: true
     steps:
+
+    - name: Set release name with date
+      id: set_release_name
+      run: echo "RELEASE_NAME=pomdp-py (dev-latest $(date +'%Y-%m-%d'))" >> $GITHUB_ENV
+
     - uses: actions/checkout@v4
       with:
         ref: 'dev-latest'
+
     - uses: actions/setup-python@v5
       with:
         python-version: '3.10'
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -29,12 +36,14 @@ jobs:
     #   run: |
     #     cd scripts
     #     ./pomdp_py_release.sh
+
     - name: Fake artifact
       run: |
         mkdir dist
-        echo "HELLO" >> test.whl
+        echo "HELLO" >> dist/test.whl
+
     - name: Release artifact
       uses: softprops/action-gh-release@v1
       with:
-        name: pomdp-py wheel
+        name: ${{ env.RELEASE_NAME }}
         files: dist/*.whl

From 60b975fe1a2392b07047ecfff5b867a25673cc14 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Wed, 6 Mar 2024 12:08:43 -0500
Subject: [PATCH 48/78] be real now - workflow works!

---
 .github/workflows/build_wheel.yml | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
index c973964f..6e540c97 100644
--- a/.github/workflows/build_wheel.yml
+++ b/.github/workflows/build_wheel.yml
@@ -31,16 +31,12 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-    # - name: Run release script
-    #   working-directory: .
-    #   run: |
-    #     cd scripts
-    #     ./pomdp_py_release.sh
 
-    - name: Fake artifact
+    - name: Run release script
+      working-directory: .
       run: |
-        mkdir dist
-        echo "HELLO" >> dist/test.whl
+        cd scripts
+        ./pomdp_py_release.sh
 
     - name: Release artifact
       uses: softprops/action-gh-release@v1

From e9d65069d422273244aa7f2bf59e22cbce7becd2 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Fri, 8 Mar 2024 15:40:53 -0500
Subject: [PATCH 49/78] rocksample minor refactor for importability

---
 .../problems/rocksample/rocksample_problem.py | 45 ++++++++++---------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/pomdp_py/problems/rocksample/rocksample_problem.py b/pomdp_py/problems/rocksample/rocksample_problem.py
index 2db7e90d..e116606c 100644
--- a/pomdp_py/problems/rocksample/rocksample_problem.py
+++ b/pomdp_py/problems/rocksample/rocksample_problem.py
@@ -502,48 +502,49 @@ def init_particles_belief(k, num_particles, init_state, belief="uniform"):
     return init_belief
 
 
-def main():
-    n, k = 5, 5
-    init_state, rock_locs = RockSampleProblem.generate_instance(n, k)
-    # # For debugging purpose
-    # n, k = 2,2
-    # rover_position = (0, 0)
-    # rock_locs = {}  # map from rock location to rock id
-    # rock_locs[(0,1)] = 0
-    # rock_locs[(1,1)] = 1
-    # rocktypes = ('good', 'good')
+def minimal_instance():
+    # A particular instance for debugging purpose
+    n, k = 2,2
+    rover_position = (0, 0)
+    rock_locs = {}  # map from rock location to rock id
+    rock_locs[(0,1)] = 0
+    rock_locs[(1,1)] = 1
+    rocktypes = ('good', 'good')
     # Ground truth state
-    # init_state = State(rover_position, rocktypes, False)
-    # belief = "uniform"
-
+    init_state = State(rover_position, rocktypes, False)
     belief = "uniform"
+    init_belief = init_particles_belief(k, 200, init_state, belief=belief)
+    rocksample = RockSampleProblem(n, k, init_state, rock_locs, init_belief)
+    return rocksample
+
+def create_instance(n, k):
+    init_state, rock_locs = RockSampleProblem.generate_instance(n, k)
 
-    init_state_copy = copy.deepcopy(init_state)
+    belief = "uniform"
 
     # init belief (uniform), represented in particles;
     # We don't factor the state here; We are also not doing any action prior.
     init_belief = init_particles_belief(k, 200, init_state, belief=belief)
 
     rocksample = RockSampleProblem(n, k, init_state, rock_locs, init_belief)
+    return rocksample
+
+
+def main():
+    rocksample = debug_instance() #create_instance(7, 8)
     rocksample.print_state()
 
     print("*** Testing POMCP ***")
     pomcp = pomdp_py.POMCP(
-        max_depth=12,
+        max_depth=30,
         discount_factor=0.95,
         num_sims=10000,
-        exploration_const=20,
+        exploration_const=5,
         rollout_policy=rocksample.agent.policy_model,
         num_visits_init=1,
     )
     tt, ttd = test_planner(rocksample, pomcp, nsteps=100, discount=0.95)
 
-    rocksample.env.state.position = init_state_copy.position
-    rocksample.env.state.rocktypes = init_state_copy.rocktypes
-    rocksample.env.state.terminal = False
-    init_belief = init_particles_belief(k, 200, rocksample.env.state, belief=belief)
-    rocksample.agent.set_belief(init_belief)
-
 
 if __name__ == "__main__":
     main()

From 0f58732800f76e43a0f06421dc44ace906aa8c5b Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Mon, 11 Mar 2024 11:18:02 -0400
Subject: [PATCH 50/78] make half_efficiency_dist tunable in rocksample

---
 .../problems/rocksample/rocksample_problem.py | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/pomdp_py/problems/rocksample/rocksample_problem.py b/pomdp_py/problems/rocksample/rocksample_problem.py
index e116606c..d41259f0 100644
--- a/pomdp_py/problems/rocksample/rocksample_problem.py
+++ b/pomdp_py/problems/rocksample/rocksample_problem.py
@@ -1,5 +1,4 @@
-"""
-RockSample(n,k) problem
+"""RockSample(n,k) problem
 
 Origin: Heuristic Search Value Iteration for POMDPs (UAI 2004)
 
@@ -24,12 +23,16 @@
     Check_i: receives a noisy observation about RockType_i
     (noise determined by eta (:math:`\eta`). eta=1 -> perfect sensor; eta=0 -> uniform)
 
-Observation: observes the property of rock i when taking Check_i.
+Observation: observes the property of rock i when taking Check_i.  The
+     observation may be noisy, depending on an efficiency parameter which
+     decreases exponentially as the distance increases between the rover and
+     rock i. 'half_efficiency_dist' influences this parameter (larger, more robust)
 
 Reward: +10 for Sample a good rock. -10 for Sampling a bad rock.
         Move to exit area +10. Other actions have no cost or reward.
 
 Initial belief: every rock has equal probability of being Good or Bad.
+
 """
 
 import pomdp_py
@@ -427,13 +430,13 @@ def print_state(self):
             string += "\n"
         print(string)
 
-    def __init__(self, n, k, init_state, rock_locs, init_belief):
+    def __init__(self, n, k, init_state, rock_locs, init_belief, half_efficiency_dist=20):
         self._n, self._k = n, k
         agent = pomdp_py.Agent(
             init_belief,
             RSPolicyModel(n, k),
             RSTransitionModel(n, rock_locs, self.in_exit_area),
-            RSObservationModel(rock_locs),
+            RSObservationModel(rock_locs, half_efficiency_dist=half_efficiency_dist),
             RSRewardModel(rock_locs, self.in_exit_area),
         )
         env = pomdp_py.Environment(
@@ -502,7 +505,7 @@ def init_particles_belief(k, num_particles, init_state, belief="uniform"):
     return init_belief
 
 
-def minimal_instance():
+def minimal_instance(**kwargs):
     # A particular instance for debugging purpose
     n, k = 2,2
     rover_position = (0, 0)
@@ -514,10 +517,10 @@ def minimal_instance():
     init_state = State(rover_position, rocktypes, False)
     belief = "uniform"
     init_belief = init_particles_belief(k, 200, init_state, belief=belief)
-    rocksample = RockSampleProblem(n, k, init_state, rock_locs, init_belief)
+    rocksample = RockSampleProblem(n, k, init_state, rock_locs, init_belief, **kwargs)
     return rocksample
 
-def create_instance(n, k):
+def create_instance(n, k, **kwargs):
     init_state, rock_locs = RockSampleProblem.generate_instance(n, k)
 
     belief = "uniform"
@@ -526,7 +529,7 @@ def create_instance(n, k):
     # We don't factor the state here; We are also not doing any action prior.
     init_belief = init_particles_belief(k, 200, init_state, belief=belief)
 
-    rocksample = RockSampleProblem(n, k, init_state, rock_locs, init_belief)
+    rocksample = RockSampleProblem(n, k, init_state, rock_locs, init_belief, **kwargs)
     return rocksample
 
 

From 9c88419e91c8aba1237c472d7ea281f0b763577b Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 14:23:18 -0400
Subject: [PATCH 51/78] add py3.12 to matrix for package ci

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 83218e23..f976d61c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v3

From fcb30719819f4f4a21c48c15a5b9e57d61b10afe Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 14:50:19 -0400
Subject: [PATCH 52/78] attempt count for more robust twine pypi push

---
 scripts/pomdp_py_release.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index c192f4df..b6ad8899 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -8,7 +8,7 @@
 #
 # Example usage:
 #
-#   ./release.sh
+#   ./release.sh <attempt_count>
 
 # Define the function.
 find_pxd_files_and_write_manifest() {
@@ -80,6 +80,10 @@ find_pxd_files_and_write_manifest ./ MANIFEST.in
 
 # Check if pomdp-py is on the right branch
 version=$(extract_package_version "$pomdp_py_path/pyproject.toml")
+if [ $# -gt 0 ]; then
+    attempt_count=$1
+    version="$version-$attempt_count"
+fi
 if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
     if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then
         echo "pomdp-py repo must be either on dev-latest or dev-$version, but not $current_branch. Abort"

From 732af8ce782354fd0e43a21ea7d3f20b3863d6fb Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 15:19:14 -0400
Subject: [PATCH 53/78] pypi deployment workflow attempt

---
 .github/workflows/pypi_deployment.yml | 52 +++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 .github/workflows/pypi_deployment.yml

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
new file mode 100644
index 00000000..e1d60037
--- /dev/null
+++ b/.github/workflows/pypi_deployment.yml
@@ -0,0 +1,52 @@
+name: pomdp-py package pypi deployment
+
+on:
+  push:
+    branches:
+      - "main"
+      - "dev-latest"
+    tags:
+      - 'v1.3.5'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pypi-publish:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/pomdp-py/
+
+    steps:
+
+    - name: Set release name with date
+      id: set_release_name
+      run: echo "RELEASE_NAME=pomdp-py (dev-latest $(date +'%Y-%m-%d'))" >> $GITHUB_ENV
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Run release script
+      working-directory: .
+      run: |
+        cd scripts
+        ./pomdp_py_release.sh ${{ secrets.RELEASE_ATTEMPT_COUNT }}
+
+    - name: Upload to PyPi
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        # This will upload stuff under dist/ to PyPI
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/

From 37da463cb25f56f25c06dc747289e79810f1b3d4 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 15:20:36 -0400
Subject: [PATCH 54/78] oops

---
 .github/workflows/pypi_deployment.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index e1d60037..12112f2c 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -33,6 +33,10 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - uses: actions/checkout@v4
+      with:
+        ref: 'dev-latest'
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

From 20e06dbd3ebb1b76330bc40ef2678cb4f4f55726 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 15:22:14 -0400
Subject: [PATCH 55/78] remove unneeded

---
 .github/workflows/pypi_deployment.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 12112f2c..e81f920c 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -25,10 +25,6 @@ jobs:
 
     steps:
 
-    - name: Set release name with date
-      id: set_release_name
-      run: echo "RELEASE_NAME=pomdp-py (dev-latest $(date +'%Y-%m-%d'))" >> $GITHUB_ENV
-
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}

From 4d8e4f520d63cf519050625f07ca6313c6c51937 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 15:29:38 -0400
Subject: [PATCH 56/78] forgot id-token

---
 .github/workflows/pypi_deployment.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index e81f920c..87d972d5 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -22,9 +22,10 @@ jobs:
     environment:
       name: testpypi
       url: https://test.pypi.org/p/pomdp-py/
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
 
     steps:
-
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}

From aa34e86a06bc265e7f2ab1fdc0e53705e261ae12 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 15:37:22 -0400
Subject: [PATCH 57/78] try test pypi token; skip pip install ask

---
 .github/workflows/pypi_deployment.yml | 2 +-
 scripts/pomdp_py_release.sh           | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 87d972d5..67b611b1 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -49,5 +49,5 @@ jobs:
       with:
         # This will upload stuff under dist/ to PyPI
         user: __token__
-        password: ${{ secrets.PYPI_TOKEN }}
+        password: ${{ secrets.TEST_PYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index b6ad8899..30d8549d 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -119,14 +119,14 @@ rm -r $pomdp_py_path/wheelhouse
 # Verification (wheel)
 echo -e "------------ verification: wheel ---------"
 pip uninstall pomdp_py
-pip install "$pomdp_py_path/dist/$fixed_wheel_name"
+pip install -y "$pomdp_py_path/dist/$fixed_wheel_name"
 python $pomdp_py_path/tests/test_all.py
 
 # Verification (source)
 echo -e "------------ verification: source ---------"
 pip uninstall pomdp_py
 cd $pomdp_py_path/dist
-pip install pomdp-py-$version.tar.gz
+pip install -y pomdp-py-$version.tar.gz
 python $pomdp_py_path/tests/test_all.py
 
 pip install twine

From e0615e6bffa1bc3013405a5f9c19f14df1c608fb Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:13:34 -0400
Subject: [PATCH 58/78] verbose

---
 .github/workflows/pypi_deployment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 67b611b1..f1e66cae 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -51,3 +51,4 @@ jobs:
         user: __token__
         password: ${{ secrets.TEST_PYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
+        verbose: true

From 6e99a24ff47d17412ba27b0e517a940f7f92a4b0 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:13:37 -0400
Subject: [PATCH 59/78] dramatic test

---
 scripts/pomdp_py_release.sh | 105 ++++++++++++++++++------------------
 1 file changed, 53 insertions(+), 52 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 30d8549d..7ebe9fa4 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -84,55 +84,56 @@ if [ $# -gt 0 ]; then
     attempt_count=$1
     version="$version-$attempt_count"
 fi
-if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
-    if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then
-        echo "pomdp-py repo must be either on dev-latest or dev-$version, but not $current_branch. Abort"
-        exit 1
-    fi
-fi
-
-echo -e "========= making release for pomdp-py $version ========="
-
-pip install setuptools
-pip install Cython
-
-# Note that we are building with pyproject.toml
-python3 setup.py build_ext --inplace
-pip install build
-python -m build
-
-# create the manylinux container
-linux_dist=manylinux2014_x86_64
-manylinux_image=quay.io/pypa/$linux_dist
-if ! does_docker_image_exist $manylinux_image; then
-   docker pull $manylinux_image
-fi
-cpv=$(get_python_version)
-wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
-command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
-docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
-rm $pomdp_py_path/dist/$wheel_name
-fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
-mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
-rm -r $pomdp_py_path/wheelhouse
-
-# Verification (wheel)
-echo -e "------------ verification: wheel ---------"
-pip uninstall pomdp_py
-pip install -y "$pomdp_py_path/dist/$fixed_wheel_name"
-python $pomdp_py_path/tests/test_all.py
-
-# Verification (source)
-echo -e "------------ verification: source ---------"
-pip uninstall pomdp_py
-cd $pomdp_py_path/dist
-pip install -y pomdp-py-$version.tar.gz
-python $pomdp_py_path/tests/test_all.py
-
-pip install twine
-echo -e "If successful, run"
-echo -e "    python3 -m twine upload --repository pypi $pomdp_py_path/dist/*"
-echo -e "to upload the release to PyPI."
-
-
-cd $user_pwd
+echo $version
+# if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
+#     if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then
+#         echo "pomdp-py repo must be either on dev-latest or dev-$version, but not $current_branch. Abort"
+#         exit 1
+#     fi
+# fi
+
+# echo -e "========= making release for pomdp-py $version ========="
+
+# pip install setuptools
+# pip install Cython
+
+# # Note that we are building with pyproject.toml
+# python3 setup.py build_ext --inplace
+# pip install build
+# python -m build
+
+# # create the manylinux container
+# linux_dist=manylinux2014_x86_64
+# manylinux_image=quay.io/pypa/$linux_dist
+# if ! does_docker_image_exist $manylinux_image; then
+#    docker pull $manylinux_image
+# fi
+# cpv=$(get_python_version)
+# wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
+# command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
+# docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
+# rm $pomdp_py_path/dist/$wheel_name
+# fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
+# mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
+# rm -r $pomdp_py_path/wheelhouse
+
+# # Verification (wheel)
+# echo -e "------------ verification: wheel ---------"
+# pip uninstall pomdp_py
+# pip install --force-reinstall "$pomdp_py_path/dist/$fixed_wheel_name"
+# python $pomdp_py_path/tests/test_all.py
+
+# # Verification (source)
+# echo -e "------------ verification: source ---------"
+# pip uninstall pomdp_py
+# cd $pomdp_py_path/dist
+# pip install --force-reinstall pomdp-py-$version.tar.gz
+# python $pomdp_py_path/tests/test_all.py
+
+# pip install twine
+# echo -e "If successful, run"
+# echo -e "    python3 -m twine upload --repository pypi $pomdp_py_path/dist/*"
+# echo -e "to upload the release to PyPI."
+
+
+# cd $user_pwd

From 3f50e9ac70dacdd647d268a8a8c96c2c9241fdb7 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:23:27 -0400
Subject: [PATCH 60/78] comment out all secrets for now

---
 .github/workflows/pypi_deployment.yml | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index f1e66cae..eecf4764 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -22,8 +22,8 @@ jobs:
     environment:
       name: testpypi
       url: https://test.pypi.org/p/pomdp-py/
-    permissions:
-      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+    # permissions:
+    #   id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
 
     steps:
     - uses: actions/setup-python@v5
@@ -42,13 +42,14 @@ jobs:
       working-directory: .
       run: |
         cd scripts
-        ./pomdp_py_release.sh ${{ secrets.RELEASE_ATTEMPT_COUNT }}
-
-    - name: Upload to PyPi
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        # This will upload stuff under dist/ to PyPI
-        user: __token__
-        password: ${{ secrets.TEST_PYPI_TOKEN }}
-        repository_url: https://test.pypi.org/legacy/
-        verbose: true
+        ./pomdp_py_release.sh 1
+        #${{ secrets.RELEASE_ATTEMPT_COUNT }}
+
+    # - name: Upload to PyPi
+    #   uses: pypa/gh-action-pypi-publish@release/v1
+    #   with:
+    #     # This will upload stuff under dist/ to PyPI
+    #     user: __token__
+    #     password: ${{ secrets.TEST_PYPI_TOKEN }}
+    #     repository_url: https://test.pypi.org/legacy/
+        # verbose: true

From 0fb2b73fe9f71a10adf6b5525234cb2053912cde Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:27:08 -0400
Subject: [PATCH 61/78] use variable

---
 .github/workflows/pypi_deployment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index eecf4764..72b6da7d 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -42,7 +42,7 @@ jobs:
       working-directory: .
       run: |
         cd scripts
-        ./pomdp_py_release.sh 1
+        ./pomdp_py_release.sh $RELEASE_ATTEMPT_COUNT
         #${{ secrets.RELEASE_ATTEMPT_COUNT }}
 
     # - name: Upload to PyPi

From 613942a18cfd44d02cf66712d92449841117ebf6 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:28:37 -0400
Subject: [PATCH 62/78] vars

---
 .github/workflows/pypi_deployment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 72b6da7d..6eacab95 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -42,7 +42,7 @@ jobs:
       working-directory: .
       run: |
         cd scripts
-        ./pomdp_py_release.sh $RELEASE_ATTEMPT_COUNT
+        ./pomdp_py_release.sh ${{ vars.RELEASE_ATTEMPT_COUNT }}
         #${{ secrets.RELEASE_ATTEMPT_COUNT }}
 
     # - name: Upload to PyPi

From bf56d3e5a2d11132d98d7eea19bf8cbc901b2f87 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:30:19 -0400
Subject: [PATCH 63/78] revert; now we may be ready

---
 .github/workflows/pypi_deployment.yml |  23 +++---
 scripts/pomdp_py_release.sh           | 104 +++++++++++++-------------
 2 files changed, 63 insertions(+), 64 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 6eacab95..b0d02f70 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -22,8 +22,8 @@ jobs:
     environment:
       name: testpypi
       url: https://test.pypi.org/p/pomdp-py/
-    # permissions:
-    #   id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
 
     steps:
     - uses: actions/setup-python@v5
@@ -43,13 +43,12 @@ jobs:
       run: |
         cd scripts
         ./pomdp_py_release.sh ${{ vars.RELEASE_ATTEMPT_COUNT }}
-        #${{ secrets.RELEASE_ATTEMPT_COUNT }}
-
-    # - name: Upload to PyPi
-    #   uses: pypa/gh-action-pypi-publish@release/v1
-    #   with:
-    #     # This will upload stuff under dist/ to PyPI
-    #     user: __token__
-    #     password: ${{ secrets.TEST_PYPI_TOKEN }}
-    #     repository_url: https://test.pypi.org/legacy/
-        # verbose: true
+
+    - name: Upload to PyPi
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        # This will upload stuff under dist/ to PyPI
+        user: __token__
+        password: ${{ secrets.TEST_PYPI_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+        verbose: true
diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 7ebe9fa4..97c617b3 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -85,55 +85,55 @@ if [ $# -gt 0 ]; then
     version="$version-$attempt_count"
 fi
 echo $version
-# if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
-#     if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then
-#         echo "pomdp-py repo must be either on dev-latest or dev-$version, but not $current_branch. Abort"
-#         exit 1
-#     fi
-# fi
-
-# echo -e "========= making release for pomdp-py $version ========="
-
-# pip install setuptools
-# pip install Cython
-
-# # Note that we are building with pyproject.toml
-# python3 setup.py build_ext --inplace
-# pip install build
-# python -m build
-
-# # create the manylinux container
-# linux_dist=manylinux2014_x86_64
-# manylinux_image=quay.io/pypa/$linux_dist
-# if ! does_docker_image_exist $manylinux_image; then
-#    docker pull $manylinux_image
-# fi
-# cpv=$(get_python_version)
-# wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
-# command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
-# docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
-# rm $pomdp_py_path/dist/$wheel_name
-# fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
-# mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
-# rm -r $pomdp_py_path/wheelhouse
-
-# # Verification (wheel)
-# echo -e "------------ verification: wheel ---------"
-# pip uninstall pomdp_py
-# pip install --force-reinstall "$pomdp_py_path/dist/$fixed_wheel_name"
-# python $pomdp_py_path/tests/test_all.py
-
-# # Verification (source)
-# echo -e "------------ verification: source ---------"
-# pip uninstall pomdp_py
-# cd $pomdp_py_path/dist
-# pip install --force-reinstall pomdp-py-$version.tar.gz
-# python $pomdp_py_path/tests/test_all.py
-
-# pip install twine
-# echo -e "If successful, run"
-# echo -e "    python3 -m twine upload --repository pypi $pomdp_py_path/dist/*"
-# echo -e "to upload the release to PyPI."
-
-
-# cd $user_pwd
+if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
+    if ! is_git_repo_on_branch $pomdp_py_path dev-latest; then
+        echo "pomdp-py repo must be either on dev-latest or dev-$version, but not $current_branch. Abort"
+        exit 1
+    fi
+fi
+
+echo -e "========= making release for pomdp-py $version ========="
+
+pip install setuptools
+pip install Cython
+
+# Note that we are building with pyproject.toml
+python3 setup.py build_ext --inplace
+pip install build
+python -m build
+
+# create the manylinux container
+linux_dist=manylinux2014_x86_64
+manylinux_image=quay.io/pypa/$linux_dist
+if ! does_docker_image_exist $manylinux_image; then
+   docker pull $manylinux_image
+fi
+cpv=$(get_python_version)
+wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
+command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
+docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
+rm $pomdp_py_path/dist/$wheel_name
+fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
+mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
+rm -r $pomdp_py_path/wheelhouse
+
+# Verification (wheel)
+echo -e "------------ verification: wheel ---------"
+pip uninstall pomdp_py
+pip install --force-reinstall "$pomdp_py_path/dist/$fixed_wheel_name"
+python $pomdp_py_path/tests/test_all.py
+
+# Verification (source)
+echo -e "------------ verification: source ---------"
+pip uninstall pomdp_py
+cd $pomdp_py_path/dist
+pip install --force-reinstall pomdp-py-$version.tar.gz
+python $pomdp_py_path/tests/test_all.py
+
+pip install twine
+echo -e "If successful, run"
+echo -e "    python3 -m twine upload --repository pypi $pomdp_py_path/dist/*"
+echo -e "to upload the release to PyPI."
+
+
+cd $user_pwd

From 4ecabce9b7ba50fc8108bbbae84fd2c68f8ba35c Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:35:25 -0400
Subject: [PATCH 64/78] attempt count shows up later

---
 scripts/pomdp_py_release.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 97c617b3..025f4509 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -80,9 +80,9 @@ find_pxd_files_and_write_manifest ./ MANIFEST.in
 
 # Check if pomdp-py is on the right branch
 version=$(extract_package_version "$pomdp_py_path/pyproject.toml")
+attempt_count=""
 if [ $# -gt 0 ]; then
-    attempt_count=$1
-    version="$version-$attempt_count"
+    attempt_count="-$1"
 fi
 echo $version
 if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
@@ -113,7 +113,7 @@ wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
 command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
 docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
 rm $pomdp_py_path/dist/$wheel_name
-fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
+fixed_wheel_name="pomdp_py-${version}${attempt_count}${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
 mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
 rm -r $pomdp_py_path/wheelhouse
 

From 6ead971d89eb661f5284210b66bce22a4ca7f8ef Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:44:06 -0400
Subject: [PATCH 65/78] fix attempt name

---
 scripts/pomdp_py_release.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 025f4509..883fc906 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -82,7 +82,7 @@ find_pxd_files_and_write_manifest ./ MANIFEST.in
 version=$(extract_package_version "$pomdp_py_path/pyproject.toml")
 attempt_count=""
 if [ $# -gt 0 ]; then
-    attempt_count="-$1"
+    attempt_count="-$1-"
 fi
 echo $version
 if ! is_git_repo_on_branch $pomdp_py_path dev-$version; then
@@ -113,14 +113,15 @@ wheel_name="pomdp_py-$version-$cpv-${cpv}-linux_x86_64.whl"
 command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
 docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
 rm $pomdp_py_path/dist/$wheel_name
-fixed_wheel_name="pomdp_py-${version}${attempt_count}${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
-mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name"
+fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
+fixed_wheel_name_with_attempt="pomdp_py-${version}-${attempt_count}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
+mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name_with_attempt"
 rm -r $pomdp_py_path/wheelhouse
 
 # Verification (wheel)
 echo -e "------------ verification: wheel ---------"
 pip uninstall pomdp_py
-pip install --force-reinstall "$pomdp_py_path/dist/$fixed_wheel_name"
+pip install --force-reinstall "$pomdp_py_path/dist/$fixed_wheel_name_with_attempt"
 python $pomdp_py_path/tests/test_all.py
 
 # Verification (source)

From 553323d3a16610405e5fbf5b238e814ae4480837 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 16:44:46 -0400
Subject: [PATCH 66/78] fix fix

---
 scripts/pomdp_py_release.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/pomdp_py_release.sh b/scripts/pomdp_py_release.sh
index 883fc906..1d2c4f55 100755
--- a/scripts/pomdp_py_release.sh
+++ b/scripts/pomdp_py_release.sh
@@ -114,7 +114,7 @@ command="auditwheel repair io/dist/${wheel_name} -w /io/wheelhouse/"
 docker run --user $(id -u):$(id -g) --mount type=bind,source=${pomdp_py_path},target=/io $manylinux_image bash -c "$command"
 rm $pomdp_py_path/dist/$wheel_name
 fixed_wheel_name="pomdp_py-${version}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
-fixed_wheel_name_with_attempt="pomdp_py-${version}-${attempt_count}-${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
+fixed_wheel_name_with_attempt="pomdp_py-${version}${attempt_count}${cpv}-${cpv}-manylinux_2_17_x86_64.$linux_dist.whl"
 mv "$pomdp_py_path/wheelhouse/$fixed_wheel_name" "$pomdp_py_path/dist/$fixed_wheel_name_with_attempt"
 rm -r $pomdp_py_path/wheelhouse
 

From e38a99d59580b8b9309b049f855c8eff30429610 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 17:13:06 -0400
Subject: [PATCH 67/78] one more attempt

---
 .github/workflows/pypi_deployment.yml | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index b0d02f70..c92319bc 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -3,7 +3,6 @@ name: pomdp-py package pypi deployment
 on:
   push:
     branches:
-      - "main"
       - "dev-latest"
     tags:
       - 'v1.3.5'
@@ -44,11 +43,22 @@ jobs:
         cd scripts
         ./pomdp_py_release.sh ${{ vars.RELEASE_ATTEMPT_COUNT }}
 
-    - name: Upload to PyPi
+    - name: Upload to Test PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
+      if: ${{ startsWith(github.ref, 'refs/heads/') }}
       with:
         # This will upload stuff under dist/ to PyPI
         user: __token__
         password: ${{ secrets.TEST_PYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
         verbose: true
+
+    - name: Upload to PyPi
+      uses: pypa/gh-action-pypi-publish@release/v1
+      if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      with:
+        # This will upload stuff under dist/ to PyPI
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+        verbose: true

From d67ef1f9caeb6b63008b2c8012b55524a6565200 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 17:24:50 -0400
Subject: [PATCH 68/78] this should do it?

---
 .github/workflows/pypi_deployment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index c92319bc..729055b8 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -61,4 +61,4 @@ jobs:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
-        verbose: true
+        skip-existing: true

From 5f827035542d7c0345af3a1ab9da58233c95b571 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 17:31:46 -0400
Subject: [PATCH 69/78] oops, wrong place

---
 .github/workflows/pypi_deployment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 729055b8..7bf040b4 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -51,7 +51,7 @@ jobs:
         user: __token__
         password: ${{ secrets.TEST_PYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
-        verbose: true
+        skip-existing: true
 
     - name: Upload to PyPi
       uses: pypa/gh-action-pypi-publish@release/v1

From 93fa947524b2440b5c8b33dbb38bb458343ee803 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 17:40:34 -0400
Subject: [PATCH 70/78] ok good now. only run the deployment action when
 tagging

---
 .github/workflows/pypi_deployment.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 7bf040b4..7f58cf12 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -2,10 +2,9 @@ name: pomdp-py package pypi deployment
 
 on:
   push:
-    branches:
-      - "dev-latest"
     tags:
       - 'v1.3.5'
+      - 'vdev-latest'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -45,7 +44,7 @@ jobs:
 
     - name: Upload to Test PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
-      if: ${{ startsWith(github.ref, 'refs/heads/') }}
+      if: ${{ startsWith(github.ref, 'refs/heads/') && github.ref == 'refs/tags/vdev-latest' }}
       with:
         # This will upload stuff under dist/ to PyPI
         user: __token__
@@ -55,7 +54,7 @@ jobs:
 
     - name: Upload to PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
-      if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      if: ${{ startsWith(github.ref, 'refs/heads/') && github.ref != 'refs/tags/vdev-latest' }}
       with:
         # This will upload stuff under dist/ to PyPI
         user: __token__

From 4f9f62e1b14ea7331951da84cec3fe0f5f89b302 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 18:14:33 -0400
Subject: [PATCH 71/78] release artifact at the end

---
 .github/workflows/pypi_deployment.yml | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 7f58cf12..28ee1ea5 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -3,7 +3,7 @@ name: pomdp-py package pypi deployment
 on:
   push:
     tags:
-      - 'v1.3.5'
+      - 'v[0-9]+(\.[0-9]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?'  # accepts semantic versioning
       - 'vdev-latest'
 
 concurrency:
@@ -24,6 +24,12 @@ jobs:
       id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
 
     steps:
+    - name: Set Version Variable
+      if: startsWith(github.ref, 'refs/tags/v')
+      env:
+        VERSION: Version ${{ github.ref_name }}
+        TRIGGER_FOR_TEST_PYPI: ${{ github.ref == 'refs/tags/vdev-latest' }}
+
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
@@ -44,7 +50,7 @@ jobs:
 
     - name: Upload to Test PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
-      if: ${{ startsWith(github.ref, 'refs/heads/') && github.ref == 'refs/tags/vdev-latest' }}
+      if: ${{ env.TRIGGER_FOR_TEST_PYPI }}
       with:
         # This will upload stuff under dist/ to PyPI
         user: __token__
@@ -54,10 +60,17 @@ jobs:
 
     - name: Upload to PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
-      if: ${{ startsWith(github.ref, 'refs/heads/') && github.ref != 'refs/tags/vdev-latest' }}
+      if: ${{ not env.TRIGGER_FOR_TEST_PYPI }}
       with:
         # This will upload stuff under dist/ to PyPI
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
         skip-existing: true
+
+    - name: Release artifact
+      uses: softprops/action-gh-release@v1
+      if: ${{ not env.TRIGGER_FOR_TEST_PYPI }}
+      with:
+        name: ${{ env.VERSION }}
+        files: dist/*.whl

From d9f492073ba38771e8aecaf84f5666d19497ebf0 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 18:25:33 -0400
Subject: [PATCH 72/78] workflow syntax fix

---
 .github/workflows/pypi_deployment.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 28ee1ea5..db24ca9c 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -24,11 +24,11 @@ jobs:
       id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
 
     steps:
-    - name: Set Version Variable
+    - name: Set Variables
       if: startsWith(github.ref, 'refs/tags/v')
-      env:
-        VERSION: Version ${{ github.ref_name }}
-        TRIGGER_FOR_TEST_PYPI: ${{ github.ref == 'refs/tags/vdev-latest' }}
+      run: |   # Run the command within a multiline string
+          echo "VERSION=\"Version ${{ github.ref_name }}\"" >> $GITHUB_ENV
+          echo "TRIGGER_FOR_TEST_PYPI=${{ github.ref == 'refs/tags/vdev-latest' }}" >> $GITHUB_ENV
 
     - uses: actions/setup-python@v5
       with:

From 465ced80f377f6d3b40b89077411653ab81635bb Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 18:26:43 -0400
Subject: [PATCH 73/78] simplify

---
 .github/workflows/pypi_deployment.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index db24ca9c..986526d5 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -3,8 +3,7 @@ name: pomdp-py package pypi deployment
 on:
   push:
     tags:
-      - 'v[0-9]+(\.[0-9]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?'  # accepts semantic versioning
-      - 'vdev-latest'
+      - 'v*'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

From 0d439d80aff31bc5ea126aab865e7b01cdacd83d Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 18:29:47 -0400
Subject: [PATCH 74/78] fix

---
 .github/workflows/pypi_deployment.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 986526d5..0915ca49 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -59,7 +59,7 @@ jobs:
 
     - name: Upload to PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
-      if: ${{ not env.TRIGGER_FOR_TEST_PYPI }}
+      if: ${{ ! env.TRIGGER_FOR_TEST_PYPI }}
       with:
         # This will upload stuff under dist/ to PyPI
         user: __token__
@@ -69,7 +69,7 @@ jobs:
 
     - name: Release artifact
       uses: softprops/action-gh-release@v1
-      if: ${{ not env.TRIGGER_FOR_TEST_PYPI }}
+      if: ${{ ! env.TRIGGER_FOR_TEST_PYPI }}
       with:
         name: ${{ env.VERSION }}
         files: dist/*.whl

From 3ea2cdef6b8ddc69778bfd62c5570a42ad08934e Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 18:37:02 -0400
Subject: [PATCH 75/78] release art

---
 .github/workflows/pypi_deployment.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/pypi_deployment.yml b/.github/workflows/pypi_deployment.yml
index 0915ca49..a074385e 100644
--- a/.github/workflows/pypi_deployment.yml
+++ b/.github/workflows/pypi_deployment.yml
@@ -69,7 +69,6 @@ jobs:
 
     - name: Release artifact
       uses: softprops/action-gh-release@v1
-      if: ${{ ! env.TRIGGER_FOR_TEST_PYPI }}
       with:
         name: ${{ env.VERSION }}
         files: dist/*.whl

From 140236e886912d94b44026c580593afeac8968c6 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 19:01:32 -0400
Subject: [PATCH 76/78] remove unnecessary readme file

---
 scripts/README.md | 31 -------------------------------
 1 file changed, 31 deletions(-)
 delete mode 100644 scripts/README.md

diff --git a/scripts/README.md b/scripts/README.md
deleted file mode 100644
index 3ef5c776..00000000
--- a/scripts/README.md
+++ /dev/null
@@ -1,31 +0,0 @@
-# pomdp-py-release
-Convenience for releasing pomdp_py
-
-
-How to release pomdp-py:
-
-1. For each python version you'd like to support, starting from low to high,
-2. activate the virtulenv for that python version
-3. Run `./release.sh <version>`
-
-   (note that if pomdp-py repo is assumed to be at `$HOME/repo/pomdp-py`.
-
-   This should generate both wheel and source builds,
-   and place them under `pomdp-py/dist` properly.
-
-4. After you've generated the wheels for all the versions of your interest,
-   push to PyPI using twine:
-
-    ```
-    python3 -m twine upload --repository pypi dist/*
-    ```
-
-
-## Repository Convention (est. 07/25/23)
-
-* When you're executing a release, besides the steps above, you should always:
-
-   1. Merge the technical content of the `dev-<version>` branch into `master`.
-   2. Create a separate PR with all the docs updates.
-
- This makes it so that the contributions of external collaborators are correctly counted by github. Otherwise, the Sphinx html doc builds will count thousands of lines of changes to the collaborators which are incorrect.

From 468c3b6eab09833ea61abe60796cc9edf3d03624 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 19:01:40 -0400
Subject: [PATCH 77/78] changelog rst

---
 CHANGELOG.rst | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 694f23d1..b81c31ec 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -3,6 +3,32 @@ Changelog
 
 Best viewed on `the website <https://h2r.github.io/pomdp-py/html/changelog.html>`_.
 
+Version 1.3.5 (03/12/2024)
+--------------------------
+* Refactored :code:`_search` in :py:mod:`~pomdp_py.algorithms.po_uct.POUCT` to be more modular
+* More fields made accessible for :py:mod:`~pomdp_py.algorithms.po_uct.POUCT`
+* Allows initializing :py:mod:`~pomdp_py.framework.basics.Agent` and :py:mod:`~pomdp_py.framework.basics.Environment` without all models ready, and provide those models later (or update them) by :code:`set_models`
+* Minor function renaming for :py:mod:`~pomdp_py.framework.basics.Option`
+* Added :py:mod:`~pomdp_py.framework.oopomdp.DictState`
+* Improved modularity and documentation for the RockSample example
+* :py:mod:`~pomdp_py.representations.distribution.particles.WeightedParticles` now hashable if frozen.
+* Changed :code:`det_dict_hash` to not use sorting
+* Added a GHA workflow  for automatically building, releasing wheels and uploading them to PyPI
+* Repo convention: default branch renamed from :code:`master` to :code:`main`, and :code:`dev-latest` is the development branch.
+
+
+WeightedParticles now hashable if frozen
+Refactor _search in POUCT for more modularity
+added a workflow that builds wheel
+*
+
+Modernize repo without introducing new features.
+* Migrate to :code:`pyproject.toml` to specify dependencies, with minimal :code:`setup.py` to build Cython extensions.
+* Separate core and optional dependencies. Optional dependencies are needed only for `pomdp_py.problems`.
+* Added CI testing to repo.
+* Added pre-commit checks to repo.
+
+
 Version 1.3.4 (01/26/2024)
 --------------------------
 * Modernize repo without introducing new features.

From a6ed82d7c1aee57a74e0a73341726df100c35671 Mon Sep 17 00:00:00 2001
From: Kaiyu Zheng <kzheng@theaiinstitute.com>
Date: Tue, 12 Mar 2024 19:04:26 -0400
Subject: [PATCH 78/78] black

---
 .../problems/rocksample/rocksample_problem.py     | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/pomdp_py/problems/rocksample/rocksample_problem.py b/pomdp_py/problems/rocksample/rocksample_problem.py
index d41259f0..2980af5a 100644
--- a/pomdp_py/problems/rocksample/rocksample_problem.py
+++ b/pomdp_py/problems/rocksample/rocksample_problem.py
@@ -430,7 +430,9 @@ def print_state(self):
             string += "\n"
         print(string)
 
-    def __init__(self, n, k, init_state, rock_locs, init_belief, half_efficiency_dist=20):
+    def __init__(
+        self, n, k, init_state, rock_locs, init_belief, half_efficiency_dist=20
+    ):
         self._n, self._k = n, k
         agent = pomdp_py.Agent(
             init_belief,
@@ -507,12 +509,12 @@ def init_particles_belief(k, num_particles, init_state, belief="uniform"):
 
 def minimal_instance(**kwargs):
     # A particular instance for debugging purpose
-    n, k = 2,2
+    n, k = 2, 2
     rover_position = (0, 0)
     rock_locs = {}  # map from rock location to rock id
-    rock_locs[(0,1)] = 0
-    rock_locs[(1,1)] = 1
-    rocktypes = ('good', 'good')
+    rock_locs[(0, 1)] = 0
+    rock_locs[(1, 1)] = 1
+    rocktypes = ("good", "good")
     # Ground truth state
     init_state = State(rover_position, rocktypes, False)
     belief = "uniform"
@@ -520,6 +522,7 @@ def minimal_instance(**kwargs):
     rocksample = RockSampleProblem(n, k, init_state, rock_locs, init_belief, **kwargs)
     return rocksample
 
+
 def create_instance(n, k, **kwargs):
     init_state, rock_locs = RockSampleProblem.generate_instance(n, k)
 
@@ -534,7 +537,7 @@ def create_instance(n, k, **kwargs):
 
 
 def main():
-    rocksample = debug_instance() #create_instance(7, 8)
+    rocksample = debug_instance()  # create_instance(7, 8)
     rocksample.print_state()
 
     print("*** Testing POMCP ***")