From d56e216f41c1dbf4562c580eedba21ebf0442c2d Mon Sep 17 00:00:00 2001
From: vodorok <gabor.kutas@ericsson.com>
Date: Thu, 7 Dec 2023 22:48:19 +0100
Subject: [PATCH] Add multiprocess store to OSX and Windows
Introducing multiprocessing to the store command for Windows and OSX
platforms.
Linux still uses processpool executor
---
codechecker_common/multiprocesspool.py | 18 ++++++++++++++
web/client/codechecker_client/blame_info.py | 17 +++----------
web/client/codechecker_client/cmd/store.py | 27 ++++-----------------
3 files changed, 27 insertions(+), 35 deletions(-)
create mode 100644 codechecker_common/multiprocesspool.py
diff --git a/codechecker_common/multiprocesspool.py b/codechecker_common/multiprocesspool.py
new file mode 100644
index 0000000000..ad23fa58be
--- /dev/null
+++ b/codechecker_common/multiprocesspool.py
@@ -0,0 +1,18 @@
+# -------------------------------------------------------------------------
+#
+# Part of the CodeChecker project, under the Apache License v2.0 with
+# LLVM Exceptions. See LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+"""
+Multiprocess compatibility module.
+"""
+
+import sys
+
+# pylint: disable=unused-import
+if sys.platform in ["darwin", "win32"]:
+ from multiprocess import Pool as MultiProcessPool
+else:
+ from concurrent.futures import ProcessPoolExecutor as MultiProcessPool
diff --git a/web/client/codechecker_client/blame_info.py b/web/client/codechecker_client/blame_info.py
index 9d02d7c70e..969ea190ae 100644
--- a/web/client/codechecker_client/blame_info.py
+++ b/web/client/codechecker_client/blame_info.py
@@ -1,14 +1,13 @@
import json
import os
-import sys
import zipfile
-from concurrent.futures import ProcessPoolExecutor
from git import Repo
from git.exc import InvalidGitRepositoryError
from typing import Dict, Iterable, Optional
from codechecker_common.logger import get_logger
+from codechecker_common.multiprocesspool import MultiProcessPool
LOG = get_logger('system')
@@ -113,17 +112,9 @@ def assemble_blame_info(
Returns the number of collected blame information.
"""
- # Currently ProcessPoolExecutor fails completely in windows.
- # Reason is most likely combination of venv and fork() not
- # being present in windows, so stuff like setting up
- # PYTHONPATH in parent CodeChecker before store is executed
- # are lost.
- if sys.platform == "win32":
- file_blame_info = __collect_blame_info_for_files(file_paths)
- else:
- with ProcessPoolExecutor() as executor:
- file_blame_info = __collect_blame_info_for_files(
- file_paths, executor.map)
+ with MultiProcessPool() as executor:
+ file_blame_info = __collect_blame_info_for_files(
+ file_paths, executor.map)
# Add blame information to the zip for the files which will be sent
# to the server if exist.
diff --git a/web/client/codechecker_client/cmd/store.py b/web/client/codechecker_client/cmd/store.py
index ad6b57e1ea..0a45194687 100644
--- a/web/client/codechecker_client/cmd/store.py
+++ b/web/client/codechecker_client/cmd/store.py
@@ -24,7 +24,6 @@
import zlib
from collections import defaultdict, namedtuple
-from concurrent.futures import ProcessPoolExecutor
from contextlib import contextmanager
from datetime import timedelta
from threading import Timer
@@ -46,6 +45,7 @@
from codechecker_common.source_code_comment_handler import \
SourceCodeCommentHandler
from codechecker_common.util import load_json
+from codechecker_common.multiprocesspool import MultiProcessPool
from codechecker_web.shared import webserver_context, host_check
from codechecker_web.shared.env import get_default_workspace
@@ -371,16 +371,8 @@ def filter_source_files_with_comments(
"""
jobs = file_report_positions.items()
- # Currently ProcessPoolExecutor fails completely in windows.
- # Reason is most likely combination of venv and fork() not
- # being present in windows, so stuff like setting up
- # PYTHONPATH in parent CodeChecker before store is executed
- # are lost.
- if sys.platform == "win32":
- return get_source_file_with_comments(jobs)
- else:
- with ProcessPoolExecutor() as executor:
- return get_source_file_with_comments(jobs, executor.map)
+ with MultiProcessPool() as executor:
+ return get_source_file_with_comments(jobs, executor.map)
def get_reports(
@@ -455,18 +447,9 @@ def assemble_zip(inputs,
LOG.debug("Processing report files ...")
- # Currently ProcessPoolExecutor fails completely in windows.
- # Reason is most likely combination of venv and fork() not
- # being present in windows, so stuff like setting up
- # PYTHONPATH in parent CodeChecker before store is executed
- # are lost.
- if sys.platform == "win32":
+ with MultiProcessPool() as executor:
analyzer_result_file_reports = parse_analyzer_result_files(
- analyzer_result_file_paths, checker_labels)
- else:
- with ProcessPoolExecutor() as executor:
- analyzer_result_file_reports = parse_analyzer_result_files(
- analyzer_result_file_paths, checker_labels, executor.map)
+ analyzer_result_file_paths, checker_labels, executor.map)
LOG.info("Processing report files done.")