added evaluate tests and improved implementation

.coveragerc

@@ -1,4 +1,3 @@
 [run]
 omit =
-    planetarium/downward.py
-    planetarium/evaluate.py
+    planetarium/downward.py

planetarium/evaluate.py

@@ -7,12 +7,84 @@
 
 
 VALIDATE = os.getenv("VALIDATE", "Validate")
+DOMAINS = {
+    "blocksworld": """;; source: https://github.com/AI-Planning/pddl-generators/blob/main/blocksworld/domain.pddl
+    ;; same as used in IPC 2023
+    ;;
+    (define (domain blocksworld)
+
+    (:requirements :strips)
+
+    (:predicates (clear ?x)
+                (on-table ?x)
+                (arm-empty)
+                (holding ?x)
+                (on ?x ?y))
+
+    (:action pickup
+    :parameters (?ob)
+    :precondition (and (clear ?ob) (on-table ?ob) (arm-empty))
+    :effect (and (holding ?ob) (not (clear ?ob)) (not (on-table ?ob))
+                (not (arm-empty))))
+
+    (:action putdown
+    :parameters  (?ob)
+    :precondition (holding ?ob)
+    :effect (and (clear ?ob) (arm-empty) (on-table ?ob)
+                (not (holding ?ob))))
+
+    (:action stack
+    :parameters  (?ob ?underob)
+    :precondition (and (clear ?underob) (holding ?ob))
+    :effect (and (arm-empty) (clear ?ob) (on ?ob ?underob)
+                (not (clear ?underob)) (not (holding ?ob))))
+
+    (:action unstack
+    :parameters  (?ob ?underob)
+    :precondition (and (on ?ob ?underob) (clear ?ob) (arm-empty))
+    :effect (and (holding ?ob) (clear ?underob)
+                (not (on ?ob ?underob)) (not (clear ?ob)) (not (arm-empty)))))
+    """,
+    "gripper": """;; source: https://github.com/AI-Planning/pddl-generators/blob/main/gripper/domain.pddl
+    (define (domain gripper)
+       (:requirements :strips)
+       (:predicates (room ?r)
+            (ball ?b)
+            (gripper ?g)
+            (at-robby ?r)
+            (at ?b ?r)
+            (free ?g)
+            (carry ?o ?g))
+
+       (:action move
+           :parameters  (?from ?to)
+           :precondition (and  (room ?from) (room ?to) (at-robby ?from))
+           :effect (and  (at-robby ?to)
+                 (not (at-robby ?from))))
+
+       (:action pick
+           :parameters (?obj ?room ?gripper)
+           :precondition  (and  (ball ?obj) (room ?room) (gripper ?gripper)
+                    (at ?obj ?room) (at-robby ?room) (free ?gripper))
+           :effect (and (carry ?obj ?gripper)
+                (not (at ?obj ?room))
+                (not (free ?gripper))))
+
+       (:action drop
+           :parameters  (?obj  ?room ?gripper)
+           :precondition  (and  (ball ?obj) (room ?room) (gripper ?gripper)
+                    (carry ?obj ?gripper) (at-robby ?room))
+           :effect (and (at ?obj ?room)
+                (free ?gripper)
+                (not (carry ?obj ?gripper)))))
+    """,
+}
 
 
 def evaluate(
     source_pddl_str: str,
     target_pddl_str: str,
-    domain_str: str,
+    domain_str: str | None = None,
     is_placeholder: bool = False,
 ) -> tuple[bool, bool, bool]:
     """Evaluate two PDDL problem descriptions for equivalence.
@@ -39,26 +111,31 @@ def evaluate(
     try:
         target_graph = builder.build(target_pddl_str)
         parseable = True
-        clean_pddl_str = problem_to_string(LenientProblemParser(target_pddl_str))
+    except Exception:
+        return parseable, solveable, equivalent
+
+    clean_pddl_str = problem_to_string(LenientProblemParser()(target_pddl_str))
+    domain_str = domain_str or DOMAINS.get(target_graph.domain)
 
+    try:
         solveable = downward.validate(
-            builder.build(domain_str),
+            domain_str,
             clean_pddl_str,
             oracle.plan_to_string(oracle.plan(target_graph)),
             VALIDATE,
         )
+    except:
+        return parseable, solveable, equivalent
 
-        if source_graph == target_graph:
-            equivalent = True
-        elif source_graph.decompose()[0] != target_graph.decompose()[0]:
-            equivalent = False
-        else:
-            equivalent = metric.equals(
-                oracle.fully_specify(source_graph, return_reduced=True),
-                oracle.fully_specify(target_graph, return_reduced=True),
-                is_placeholder=is_placeholder,
-            )
-    except Exception:
-        pass
+    if source_graph == target_graph:
+        equivalent = True
+    elif source_graph.decompose()[0] != target_graph.decompose()[0]:
+        equivalent = False
+    else:
+        equivalent = metric.equals(
+            oracle.fully_specify(source_graph, return_reduced=True),
+            oracle.fully_specify(target_graph, return_reduced=True),
+            is_placeholder=is_placeholder,
+        )
 
     return parseable, solveable, equivalent

tests/test_evaluate.py

@@ -0,0 +1,59 @@
+from itertools import product
+
+from planetarium import evaluate
+
+from .test_oracle import (
+    blocksworld_underspecified,
+    blocksworld_missing_clears,
+    blocksworld_missing_ontables,
+    blocksworld_fully_specified,
+)
+
+
+class TestEvaluate:
+    """
+    Test suite for the evaluation of PDDL problem descriptions.
+    """
+
+    def test_evaluate_equivalent(
+        self,
+        subtests,
+        blocksworld_missing_clears,
+        blocksworld_fully_specified,
+        blocksworld_missing_ontables,
+    ):
+        """
+        Test if the evaluation of PDDL problem descriptions is correct.
+        """
+        descs = [
+            ("blocksworld_missing_clears", blocksworld_missing_clears),
+            ("blocksworld_fully_specified", blocksworld_fully_specified),
+            ("blocksworld_missing_ontables", blocksworld_missing_ontables),
+        ]
+        for (name1, desc1), (name2, desc2) in product(descs, descs):
+            with subtests.test(f"{name1} equals {name2}"):
+                assert all(evaluate.evaluate(desc1, desc2))
+
+    def test_evaluate_inequivalent(
+        self,
+        subtests,
+        blocksworld_missing_clears,
+        blocksworld_fully_specified,
+        blocksworld_missing_ontables,
+        blocksworld_underspecified,
+    ):
+        """
+        Test if the evaluation of PDDL problem descriptions is correct.
+        """
+        descs = [
+            ("blocksworld_missing_clears", blocksworld_missing_clears),
+            ("blocksworld_fully_specified", blocksworld_fully_specified),
+            ("blocksworld_missing_ontables", blocksworld_missing_ontables),
+        ]
+        for name1, desc1 in descs:
+            with subtests.test(f"{name1} not equals blocksworld_underspecified"):
+                assert evaluate.evaluate(desc1, blocksworld_underspecified) == (
+                    True,
+                    True,
+                    False,
+                )