Merge pull request Ericsson#4118 from vodorok/osx_and_windows_multp

[cmd][store] Add `multiprocess`-based store to MacOS and Windows

codechecker_common/multiprocesspool.py

@@ -0,0 +1,18 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+"""
+Multiprocess compatibility module.
+"""
+
+import sys
+
+# pylint: disable=unused-import
+if sys.platform in ["darwin", "win32"]:
+    from multiprocess import Pool as MultiProcessPool
+else:
+    from concurrent.futures import ProcessPoolExecutor as MultiProcessPool

web/client/codechecker_client/blame_info.py

@@ -1,14 +1,13 @@
 import json
 import os
-import sys
 import zipfile
 
-from concurrent.futures import ProcessPoolExecutor
 from git import Repo
 from git.exc import InvalidGitRepositoryError
 from typing import Dict, Iterable, Optional
 
 from codechecker_common.logger import get_logger
+from codechecker_common.multiprocesspool import MultiProcessPool
 
 LOG = get_logger('system')
 
@@ -113,17 +112,9 @@ def assemble_blame_info(
 
     Returns the number of collected blame information.
     """
-    # Currently ProcessPoolExecutor fails completely in windows.
-    # Reason is most likely combination of venv and fork() not
-    # being present in windows, so stuff like setting up
-    # PYTHONPATH in parent CodeChecker before store is executed
-    # are lost.
-    if sys.platform == "win32":
-        file_blame_info = __collect_blame_info_for_files(file_paths)
-    else:
-        with ProcessPoolExecutor() as executor:
-            file_blame_info = __collect_blame_info_for_files(
-                file_paths, executor.map)
+    with MultiProcessPool() as executor:
+        file_blame_info = __collect_blame_info_for_files(
+            file_paths, executor.map)
 
     # Add blame information to the zip for the files which will be sent
     # to the server if exist.

web/client/codechecker_client/cmd/store.py

@@ -24,7 +24,6 @@
 import zlib
 
 from collections import defaultdict, namedtuple
-from concurrent.futures import ProcessPoolExecutor
 from contextlib import contextmanager
 from datetime import timedelta
 from threading import Timer
@@ -46,6 +45,7 @@
 from codechecker_common.source_code_comment_handler import \
     SourceCodeCommentHandler
 from codechecker_common.util import load_json
+from codechecker_common.multiprocesspool import MultiProcessPool
 
 from codechecker_web.shared import webserver_context, host_check
 from codechecker_web.shared.env import get_default_workspace
@@ -371,16 +371,8 @@ def filter_source_files_with_comments(
     """
     jobs = file_report_positions.items()
 
-    # Currently ProcessPoolExecutor fails completely in windows.
-    # Reason is most likely combination of venv and fork() not
-    # being present in windows, so stuff like setting up
-    # PYTHONPATH in parent CodeChecker before store is executed
-    # are lost.
-    if sys.platform == "win32":
-        return get_source_file_with_comments(jobs)
-    else:
-        with ProcessPoolExecutor() as executor:
-            return get_source_file_with_comments(jobs, executor.map)
+    with MultiProcessPool() as executor:
+        return get_source_file_with_comments(jobs, executor.map)
 
 
 def get_reports(
@@ -455,18 +447,9 @@ def assemble_zip(inputs,
 
     LOG.debug("Processing report files ...")
 
-    # Currently ProcessPoolExecutor fails completely in windows.
-    # Reason is most likely combination of venv and fork() not
-    # being present in windows, so stuff like setting up
-    # PYTHONPATH in parent CodeChecker before store is executed
-    # are lost.
-    if sys.platform == "win32":
+    with MultiProcessPool() as executor:
         analyzer_result_file_reports = parse_analyzer_result_files(
-            analyzer_result_file_paths, checker_labels)
-    else:
-        with ProcessPoolExecutor() as executor:
-            analyzer_result_file_reports = parse_analyzer_result_files(
-                 analyzer_result_file_paths, checker_labels, executor.map)
+             analyzer_result_file_paths, checker_labels, executor.map)
 
     LOG.info("Processing report files done.")