diff --git a/codechecker_common/multiprocesspool.py b/codechecker_common/multiprocesspool.py new file mode 100644 index 0000000000..ad23fa58be --- /dev/null +++ b/codechecker_common/multiprocesspool.py @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- +""" +Multiprocess compatibility module. +""" + +import sys + +# pylint: disable=unused-import +if sys.platform in ["darwin", "win32"]: + from multiprocess import Pool as MultiProcessPool +else: + from concurrent.futures import ProcessPoolExecutor as MultiProcessPool diff --git a/web/client/codechecker_client/blame_info.py b/web/client/codechecker_client/blame_info.py index 9d02d7c70e..969ea190ae 100644 --- a/web/client/codechecker_client/blame_info.py +++ b/web/client/codechecker_client/blame_info.py @@ -1,14 +1,13 @@ import json import os -import sys import zipfile -from concurrent.futures import ProcessPoolExecutor from git import Repo from git.exc import InvalidGitRepositoryError from typing import Dict, Iterable, Optional from codechecker_common.logger import get_logger +from codechecker_common.multiprocesspool import MultiProcessPool LOG = get_logger('system') @@ -113,17 +112,9 @@ def assemble_blame_info( Returns the number of collected blame information. """ - # Currently ProcessPoolExecutor fails completely in windows. - # Reason is most likely combination of venv and fork() not - # being present in windows, so stuff like setting up - # PYTHONPATH in parent CodeChecker before store is executed - # are lost. - if sys.platform == "win32": - file_blame_info = __collect_blame_info_for_files(file_paths) - else: - with ProcessPoolExecutor() as executor: - file_blame_info = __collect_blame_info_for_files( - file_paths, executor.map) + with MultiProcessPool() as executor: + file_blame_info = __collect_blame_info_for_files( + file_paths, executor.map) # Add blame information to the zip for the files which will be sent # to the server if exist. diff --git a/web/client/codechecker_client/cmd/store.py b/web/client/codechecker_client/cmd/store.py index ad6b57e1ea..0a45194687 100644 --- a/web/client/codechecker_client/cmd/store.py +++ b/web/client/codechecker_client/cmd/store.py @@ -24,7 +24,6 @@ import zlib from collections import defaultdict, namedtuple -from concurrent.futures import ProcessPoolExecutor from contextlib import contextmanager from datetime import timedelta from threading import Timer @@ -46,6 +45,7 @@ from codechecker_common.source_code_comment_handler import \ SourceCodeCommentHandler from codechecker_common.util import load_json +from codechecker_common.multiprocesspool import MultiProcessPool from codechecker_web.shared import webserver_context, host_check from codechecker_web.shared.env import get_default_workspace @@ -371,16 +371,8 @@ def filter_source_files_with_comments( """ jobs = file_report_positions.items() - # Currently ProcessPoolExecutor fails completely in windows. - # Reason is most likely combination of venv and fork() not - # being present in windows, so stuff like setting up - # PYTHONPATH in parent CodeChecker before store is executed - # are lost. - if sys.platform == "win32": - return get_source_file_with_comments(jobs) - else: - with ProcessPoolExecutor() as executor: - return get_source_file_with_comments(jobs, executor.map) + with MultiProcessPool() as executor: + return get_source_file_with_comments(jobs, executor.map) def get_reports( @@ -455,18 +447,9 @@ def assemble_zip(inputs, LOG.debug("Processing report files ...") - # Currently ProcessPoolExecutor fails completely in windows. - # Reason is most likely combination of venv and fork() not - # being present in windows, so stuff like setting up - # PYTHONPATH in parent CodeChecker before store is executed - # are lost. - if sys.platform == "win32": + with MultiProcessPool() as executor: analyzer_result_file_reports = parse_analyzer_result_files( - analyzer_result_file_paths, checker_labels) - else: - with ProcessPoolExecutor() as executor: - analyzer_result_file_reports = parse_analyzer_result_files( - analyzer_result_file_paths, checker_labels, executor.map) + analyzer_result_file_paths, checker_labels, executor.map) LOG.info("Processing report files done.")