[tools] Fix hanging of docs.modm.io generator

modm-io · Dec 14, 2024 · 5cc3617 · 5cc3617
1 parent 8e3c047
commit 5cc3617
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 12 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -384,7 +384,7 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
     container:
-      image: ghcr.io/modm-ext/modm-build-base:2024-12-01
+      image: ghcr.io/modm-ext/modm-build-base:2024-12-14
     steps:
       - name: Check out repository
         uses: actions/checkout@v4

diff --git a/tools/scripts/docs_modm_io_generator.py b/tools/scripts/docs_modm_io_generator.py
@@ -145,17 +145,20 @@ def main():
         print("Starting to generate documentation...")
         template_overview(output_dir, device_list, board_list, template_path)
         print(f"... for {len(device_list) + len(board_list)} devices, estimated memory footprint is {len(device_list)*70+2000} MB")
-        with ThreadPool(args.jobs) as pool:
-            # We can only pass one argument to pool.map
-            devices = [f'python3 {filepath} --target-job "{modm_path}|{tempdir}|{dev}||{args.deduplicate}|{args.compress}"' for dev in device_list]
-            devices += [f'python3 {filepath} --target-job "{modm_path}|{tempdir}|{dev}|{brd}|{args.deduplicate}|{args.compress}"' for (brd, dev) in board_list]
-            devices = list(set(devices))
-            # Run the first generation first so that the other jobs can already deduplicate properly
-            results = [subprocess.call(devices[0], shell=True)]
-            results += pool.map(lambda d: subprocess.call(d, shell=True), devices[1:])
-            # remove all the hash files
-            for file in (output_dir / "develop/api").glob("*.hash"):
-                file.unlink()
+
+        # We can only pass one argument to pool.map
+        devices = [f'python3 {filepath} --target-job "{modm_path}|{tempdir}|{dev}||{args.deduplicate}|{args.compress}"' for dev in device_list]
+        devices += [f'python3 {filepath} --target-job "{modm_path}|{tempdir}|{dev}|{brd}|{args.deduplicate}|{args.compress}"' for (brd, dev) in board_list]
+        devices = list(set(devices))
+        # Run the first generation first so that the other jobs can already deduplicate properly
+        results = [subprocess.call(devices[0], shell=True)]
+        (tempdir / "tasks").write_text("\n".join(devices[1:]))
+        subprocess.call(f"cd {tempdir}; parallel -j {args.jobs} --joblog joblog -a tasks", shell=True)
+        results += [int(line.split("\t")[6]) for line in (tempdir / "joblog").read_text().splitlines()[1:]]
+
+        # remove all the hash files
+        for file in (output_dir / "develop/api").glob("*.hash"):
+            file.unlink()
         if args.compress:
             print("Zipping docs ...")
             # Zipping is *much* faster via command line than via python!