ci: add IPC problems using UPF

.github/workflows/aries.yml

@@ -94,10 +94,26 @@ jobs:
         run: ./just ci-up-solve
       - name: Validator tests
         run: ./just ci-up-val
+
+
+  ipc-tests:
+    name: IPC Tests
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Submodules initialization
+        run: git submodule update --init  # get our own copy of unified-planing repos
+      - name: Install python dependencies
+        run: python3 -m pip install -r planning/unified/requirements.txt "numpy<2"
+      - name: Install just
+        run: curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to .
+      - name: Resolution
+        run: ./just ci-ipc
 
 
   tests: # Meta-job that only requires all test-jobs to pass
-    needs: [ lints, unit-tests, integration-tests, unified-planning-api, unified-planning-integration ]
+    needs: [ lints, unit-tests, integration-tests, unified-planning-api, unified-planning-integration, ipc-tests ]
     runs-on: ubuntu-latest
     steps:
       - run: true

ci/ipc.py

@@ -0,0 +1,247 @@
+from pathlib import Path
+import re
+import subprocess  # nosec: B404
+import sys
+from tempfile import NamedTemporaryFile
+import time
+from typing import List, Optional, Tuple
+
+from unified_planning.engines.results import PlanGenerationResultStatus as Status
+from unified_planning.io.pddl_reader import PDDLReader
+from unified_planning.plans import Plan
+from unified_planning.shortcuts import OneshotPlanner
+
+
+# ============================================================================ #
+#                                   Constants                                  #
+# ============================================================================ #
+
+
+ESC_TABLE = {
+    "black": 30,
+    "red": 31,
+    "green": 32,
+    "yellow": 33,
+    "blue": 34,
+    "purple": 35,
+    "cyan": 36,
+    "white": 37,
+    "Black": 40,
+    "Red": 41,
+    "Green": 42,
+    "Yellow": 43,
+    "Blue": 44,
+    "Purple": 45,
+    "Cyan": 46,
+    "White": 47,
+    "bold": 1,
+    "light": 2,
+    "blink": 5,
+    "invert": 7,
+}
+VALID_STATUS = {Status.SOLVED_SATISFICING, Status.SOLVED_OPTIMALLY}
+
+
+# ============================================================================ #
+#                                     Utils                                    #
+# ============================================================================ #
+
+
+def write(text: str = "", **markup: bool) -> None:
+    """Write text with ANSI escape sequences for formatting."""
+    for name in markup:
+        if name not in ESC_TABLE:
+            raise ValueError(f"Unknown markup: {name}")
+    esc = [ESC_TABLE[name] for name, value in markup.items() if value]
+    if esc:
+        text = "".join(f"\x1b[{cod}m" for cod in esc) + text + "\x1b[0m"
+    print(text)
+
+
+def separator(
+    sep: str = "=",
+    title: Optional[str] = None,
+    width: int = 80,
+    before: str = "\n",
+    after: str = "\n",
+    **markup: bool,
+) -> None:
+    """Print a separator line with optional title."""
+    if title is None:
+        line = sep * (width // len(sep))
+    else:
+        n = max((width - len(title) - 2) // (2 * len(sep)), 1)
+        fill = sep * n
+        line = f"{fill} {title} {fill}"
+    if len(line) + len(sep.rstrip()) <= width:
+        line += sep.rstrip()
+    write(f"{before}{line}{after}", **markup)
+
+
+def big_separator(
+    sep: str = "=",
+    title: Optional[str] = None,
+    width: int = 80,
+    **markup: bool,
+):
+    """Print a big separator line with optional title."""
+    separator(sep=sep, title=None, width=width, before="\n", after="", **markup)
+    separator(sep=" ", title=title, width=width, before="", after="", **markup)
+    separator(sep=sep, title=None, width=width, before="", after="\n", **markup)
+
+
+# ============================================================================ #
+#                                  Validators                                  #
+# ============================================================================ #
+
+
+def extract_plan_for_val(plan: Plan, dom: Path, pb: Path, out: Path) -> None:
+    """Reformat a plan to be validated by Val."""
+
+    domain_content = dom.read_text()
+    problem_content = pb.read_text()
+
+    def fix_upf_shit(name: str) -> str:
+        name_alt = name.replace("-", "_")
+        if name in domain_content or name in problem_content:
+            return name
+        if name_alt in domain_content or name_alt in problem_content:
+            return name_alt
+        raise ValueError(f"Name {name} not found in domain or problem")
+
+    def format_line(line: str) -> str:
+        if line.startswith(";"):
+            return line
+
+        regex = r"^\s*(?:(?P<time>\d+\.\d+):\s?)?(?P<name>[\w-]+)(?:\((?P<params>[\w, -]+)\))?\s?(?:\[(?P<duration>\d+\.\d+)\])?"  # pylint:disable=line-too-long # noqa: E501
+        if not (match := re.match(regex, line)):
+            return line
+        groups = match.groupdict()
+
+        new_line = ""
+        if groups["time"]:
+            new_line += f"{groups['time']}: "
+        new_line += "("
+        new_line += fix_upf_shit(groups["name"])
+        if groups["params"]:
+            new_line += " " + " ".join(map(fix_upf_shit, groups["params"].split(", ")))
+        new_line += ")"
+        if groups["duration"]:
+            new_line += f" [{groups['duration']}]"
+        return new_line
+
+    lines = str(plan).splitlines()
+    out.write_text("\n".join(map(format_line, lines[1:])))
+
+
+def extract_max_depth(log_file: Path) -> int:
+    """Extract the max depth used by the planner from the logs."""
+    depth = 0
+    for line in log_file.read_text().splitlines():
+        if "Solving with depth" in line:
+            depth = int(line.split("Solving with depth")[-1])
+    return depth
+
+
+def validate_plan_with_val(pb: Path, dom: Path, plan: Path) -> bool:
+    """Validate a plan using Val."""
+    cmd = f"./planning/ext/val-pddl {dom.as_posix()} {pb.as_posix()} {plan.as_posix()}"
+    return subprocess.run(cmd, shell=True, check=False).returncode == 0  # nosec: B602
+
+
+# ============================================================================ #
+#                                     Main                                     #
+# ============================================================================ #
+
+
+pb_folders = Path(__file__).parent.parent / "planning/problems/upf"
+problems = sorted(pb_folders.iterdir(), key=lambda f: f.stem)
+
+valid: List[str] = []
+invalid: List[str] = []
+unsolved: List[Tuple[str, str]] = []
+
+try:
+    for i, folder in enumerate(problems):
+        separator(
+            title=f"Problem {folder.stem} ({i + 1}/{len(problems)})",
+            bold=True,
+            blue=True,
+        )
+
+        try:
+            domain = folder / "domain.pddl"
+            problem = folder / "problem.pddl"
+            upf_pb = PDDLReader().parse_problem(domain, problem)
+            max_depth = int((folder / "max_depth.txt").read_text())
+
+            with NamedTemporaryFile("w+", encoding="utf-8") as output:
+                with OneshotPlanner(
+                    name="aries",
+                    params={"max-depth": max_depth},
+                ) as planner:
+                    # Problem Kind
+                    write("Problem Kind", cyan=True)
+                    write(str(upf_pb.kind), light=True)
+                    write()
+
+                    # Unsupported features
+                    unsupported = upf_pb.kind.features.difference(
+                        planner.supported_kind().features
+                    )
+                    if unsupported:
+                        write("Unsupported Features", cyan=True)
+                        write("\n".join(unsupported), light=True)
+                        write()
+
+                    # Resolution
+                    start = time.time()
+                    result = planner.solve(upf_pb, output_stream=output)
+                    write("Resolution status", cyan=True)
+                    write(f"Elapsed time: {time.time() - start:.3f} s", light=True)
+                    write(f"Status: {result.status}", light=True)
+
+                    # Solved problem
+                    if result.status in VALID_STATUS:
+                        write("\nValidating plan by Val", cyan=True)
+                        out_path = Path(output.name)
+                        extract_plan_for_val(result.plan, domain, problem, out_path)
+                        if validate_plan_with_val(problem, domain, out_path):
+                            write("Plan is valid", bold=True, green=True)
+                            valid.append(folder.stem)
+                        else:
+                            write("Plan is invalid", bold=True, red=True)
+                            invalid.append(folder.stem)
+
+                    # Unsolved problem
+                    else:
+                        unsolved.append((folder.stem, result.status.name))
+                        write("Unsolved problem", bold=True, red=True)
+
+        except Exception as e:  # pylint: disable=broad-except
+            unsolved.append((folder.stem, "Error"))
+            write(f"Error: {e}", bold=True, red=True)
+
+except KeyboardInterrupt:
+    pass
+finally:
+    # Summary
+    if valid:
+        big_separator(title=f"Valid: {len(valid)}", bold=True, green=True)
+        for res in valid:
+            print(res)
+
+    if invalid:
+        big_separator(title=f"Invalid: {len(invalid)}", bold=True, red=True)
+        for res in invalid:
+            print(res)
+
+    if unsolved:
+        big_separator(title=f"Unsolved: {len(unsolved)}", bold=True, red=True)
+        offset = max(map(len, map(lambda t: t[0], unsolved))) + 3
+        for res, reason in unsolved:
+            print(f"{res:<{offset}} {reason}")
+
+    EXIT_CODE = 0 if not invalid and not unsolved else 1
+    big_separator(title=f"End with code {EXIT_CODE}", bold=True)
+    sys.exit(EXIT_CODE)

justfile

@@ -2,7 +2,7 @@
 set dotenv-load := true
 
 
-ci: ci-up-solve ci-up-val
+ci: ci-up-solve ci-up-val ci-ipc
 
 # Run planning tests for UP integration
 ci-up-solve:
@@ -12,6 +12,10 @@ ci-up-solve:
 ci-up-val:
     python3 planning/unified/deps/unified-planning/up_test_cases/report.py aries-val -e up_aries_tests
 
+# Run resolution tests on IPC problems
+ci-ipc:
+    ARIES_UP_ASSUME_REALS_ARE_INTS=true python3 ci/ipc.py
+
 # Solve a UP test case
 up-solve problem:
     python3 planning/unified/scripts/cli.py {{problem}}

planning/problems/upf/ipc1998-gripper-round1-strips/domain.pddl

@@ -0,0 +1,16 @@
+(define (domain strips_gripper_x_1-domain)
+ (:requirements :strips :typing)
+ (:predicates (room ?r - object) (ball ?b - object) (gripper ?g - object) (at_robby ?r - object) (at_ ?b - object ?r - object) (free ?g - object) (carry ?o - object ?g - object))
+ (:action move
+  :parameters ( ?from - object ?to - object)
+  :precondition (and (room ?from) (room ?to) (at_robby ?from))
+  :effect (and (at_robby ?to) (not (at_robby ?from))))
+ (:action pick
+  :parameters ( ?obj - object ?room - object ?gripper - object)
+  :precondition (and (ball ?obj) (room ?room) (gripper ?gripper) (at_ ?obj ?room) (at_robby ?room) (free ?gripper))
+  :effect (and (carry ?obj ?gripper) (not (at_ ?obj ?room)) (not (free ?gripper))))
+ (:action drop
+  :parameters ( ?obj - object ?room - object ?gripper - object)
+  :precondition (and (ball ?obj) (room ?room) (gripper ?gripper) (carry ?obj ?gripper) (at_robby ?room))
+  :effect (and (at_ ?obj ?room) (free ?gripper) (not (carry ?obj ?gripper))))
+)

planning/problems/upf/ipc1998-gripper-round1-strips/max_depth.txt

@@ -0,0 +1 @@
+4

planning/problems/upf/ipc1998-gripper-round1-strips/problem.pddl

@@ -0,0 +1,8 @@
+(define (problem strips_gripper_x_1-problem)
+ (:domain strips_gripper_x_1-domain)
+ (:objects
+   rooma roomb ball4 ball3 ball2 ball1 left right - object
+ )
+ (:init (room rooma) (room roomb) (ball ball4) (ball ball3) (ball ball2) (ball ball1) (at_robby rooma) (free left) (free right) (at_ ball4 rooma) (at_ ball3 rooma) (at_ ball2 rooma) (at_ ball1 rooma) (gripper left) (gripper right))
+ (:goal (and (at_ ball4 roomb) (at_ ball3 roomb) (at_ ball2 roomb) (at_ ball1 roomb)))
+)

planning/problems/upf/ipc1998-logistics-round2-strips/domain.pddl

@@ -0,0 +1,28 @@
+(define (domain strips_log_y_1-domain)
+ (:requirements :strips :typing)
+ (:predicates (obj ?obj - object) (truck ?truck - object) (location ?loc - object) (airplane ?airplane - object) (city ?city - object) (airport ?airport - object) (at_ ?obj - object ?loc - object) (in ?obj1 - object ?obj2 - object) (in_city ?obj - object ?city - object))
+ (:action load_truck
+  :parameters ( ?obj - object ?truck - object ?loc - object)
+  :precondition (and (obj ?obj) (truck ?truck) (location ?loc) (at_ ?truck ?loc) (at_ ?obj ?loc))
+  :effect (and (not (at_ ?obj ?loc)) (in ?obj ?truck)))
+ (:action load_airplane
+  :parameters ( ?obj - object ?airplane - object ?loc - object)
+  :precondition (and (obj ?obj) (airplane ?airplane) (location ?loc) (at_ ?obj ?loc) (at_ ?airplane ?loc))
+  :effect (and (not (at_ ?obj ?loc)) (in ?obj ?airplane)))
+ (:action unload_truck
+  :parameters ( ?obj - object ?truck - object ?loc - object)
+  :precondition (and (obj ?obj) (truck ?truck) (location ?loc) (at_ ?truck ?loc) (in ?obj ?truck))
+  :effect (and (not (in ?obj ?truck)) (at_ ?obj ?loc)))
+ (:action unload_airplane
+  :parameters ( ?obj - object ?airplane - object ?loc - object)
+  :precondition (and (obj ?obj) (airplane ?airplane) (location ?loc) (in ?obj ?airplane) (at_ ?airplane ?loc))
+  :effect (and (not (in ?obj ?airplane)) (at_ ?obj ?loc)))
+ (:action drive_truck
+  :parameters ( ?truck - object ?loc_from - object ?loc_to - object ?city - object)
+  :precondition (and (truck ?truck) (location ?loc_from) (location ?loc_to) (city ?city) (at_ ?truck ?loc_from) (in_city ?loc_from ?city) (in_city ?loc_to ?city))
+  :effect (and (not (at_ ?truck ?loc_from)) (at_ ?truck ?loc_to)))
+ (:action fly_airplane
+  :parameters ( ?airplane - object ?loc_from - object ?loc_to - object)
+  :precondition (and (airplane ?airplane) (airport ?loc_from) (airport ?loc_to) (at_ ?airplane ?loc_from))
+  :effect (and (not (at_ ?airplane ?loc_from)) (at_ ?airplane ?loc_to)))
+)

planning/problems/upf/ipc1998-logistics-round2-strips/max_depth.txt

@@ -0,0 +1 @@
+3

planning/problems/upf/ipc1998-logistics-round2-strips/problem.pddl

@@ -0,0 +1,8 @@
+(define (problem strips_log_y_1-problem)
+ (:domain strips_log_y_1-domain)
+ (:objects
+   package3 package2 package1 city5 city4 city3 city2 city1 truck5 truck4 truck3 truck2 truck1 plane2 plane1 city5_1 city4_1 city3_1 city2_1 city1_1 city5_2 city4_2 city3_2 city2_2 city1_2 - object
+ )
+ (:init (obj package3) (obj package2) (obj package1) (city city5) (city city4) (city city3) (city city2) (city city1) (truck truck5) (truck truck4) (truck truck3) (truck truck2) (truck truck1) (airplane plane2) (airplane plane1) (location city5_1) (location city4_1) (location city3_1) (location city2_1) (location city1_1) (airport city5_2) (location city5_2) (airport city4_2) (location city4_2) (airport city3_2) (location city3_2) (airport city2_2) (location city2_2) (airport city1_2) (location city1_2) (in_city city5_2 city5) (in_city city5_1 city5) (in_city city4_2 city4) (in_city city4_1 city4) (in_city city3_2 city3) (in_city city3_1 city3) (in_city city2_2 city2) (in_city city2_1 city2) (in_city city1_2 city1) (in_city city1_1 city1) (at_ plane2 city1_2) (at_ plane1 city2_2) (at_ truck5 city5_1) (at_ truck4 city4_1) (at_ truck3 city3_1) (at_ truck2 city2_1) (at_ truck1 city1_1) (at_ package3 city1_1) (at_ package2 city1_2) (at_ package1 city4_1))
+ (:goal (and (at_ package3 city1_2) (at_ package2 city1_1) (at_ package1 city3_2)))
+)