Environment initialization for binaries

CodeChecker calls various binaries during analysis: clang, clang-tidy,
clang-extdef-mapping etc.
If the binary is delivered within the CodeChecker package,
LD_LIBRARY_PATH needs to be extended with it.

Otherwise the environment of the caller shell should be used
for excuting binaries.

Author: dkrupp

analyzer/codechecker_analyzer/analyzer.py

@@ -247,7 +247,7 @@ def perform_analysis(args, skip_handlers, rs_handler: ReviewStatusHandler,
                 enabled_checkers[analyzer].append(check)
 
         version = analyzer_types.supported_analyzers[analyzer] \
-            .get_binary_version(context.get_analyzer_env(analyzer))
+            .get_binary_version()
         metadata_info['analyzer_statistics']['version'] = version
 
         metadata_tool['analyzers'][analyzer] = metadata_info

analyzer/codechecker_analyzer/analyzer_context.py

@@ -21,6 +21,7 @@
 from codechecker_common.checker_labels import CheckerLabels
 from codechecker_common.singleton import Singleton
 from codechecker_common.util import load_json
+from pathlib import Path
 
 from . import env
 
@@ -62,8 +63,13 @@ def __init__(self):
         self.__package_build_date = None
         self.__package_git_hash = None
         self.__analyzers = {}
-        self.__analyzer_envs = {}
+
+        # CodeChecker's current runtime environment
         self.__cc_env = None
+        # cc_env extended with packaged LD_LIBRARY_PATH for packaged binaries
+        self.__package_env = None
+        # Original caller environment of CodeChecker for external binaries
+        self.__original_env = None
 
         self.logger_lib_dir_path = os.path.join(
             self._data_files_dir_path, 'ld_logger', 'lib')
@@ -155,6 +161,9 @@ def __init_env(self):
             self.env_vars['cc_logger_compiles'])
         self.ld_preload = os.environ.get(self.env_vars['ld_preload'])
         self.ld_lib_path = self.env_vars['env_ld_lib_path']
+        self.__original_env = env.get_original_env()
+        self.__package_env = env.extend(self.path_env_extra,
+                                        self.ld_lib_path_extra)
 
     def __set_version(self):
         """
@@ -186,6 +195,29 @@ def __set_version(self):
                 logger.DEBUG_ANALYZER):
             self.__package_git_tag = package_git_dirtytag
 
+    def get_env_for_bin(self, binary):
+        """
+        binary must be a binary with full path
+
+        Returns the correct environment for binaries called by CodeChecker.
+        For binaries packaged with CodeChecker the LD_LIBRARY path is extended.
+        For non-packaged binaries, the original calling environment
+        is returned.
+        """
+        bin_path = Path(binary).resolve()
+        if not bin_path.exists():
+            LOG.error("Binary %s not found", binary)
+            return None
+
+        codechecker_dir = Path(self._data_files_dir_path)
+
+        if str(bin_path).startswith(str(codechecker_dir)):
+            LOG.debug("Package env is returned for %s", bin_path)
+            return self.__package_env
+        else:
+            LOG.debug("Original env is returned for %s", bin_path)
+            return self.__original_env
+
     def __populate_analyzers(self):
         """ Set analyzer binaries for each registered analyzers. """
         cc_env = None
@@ -198,9 +230,8 @@ def __populate_analyzers(self):
         compiler_binaries = self.pckg_layout.get('analyzers')
         for name, value in compiler_binaries.items():
             if name in env_var_bin:
-                # For non-packaged analyzers the original env is set.
+                # env_var_bin has priority over package config and PATH
                 self.__analyzers[name] = env_var_bin[name]
-                self.__analyzer_envs[name] = env.get_original_env()
                 continue
 
             if analyzer_from_path:
@@ -210,10 +241,6 @@ def __populate_analyzers(self):
                 # Check if it is a package relative path.
                 self.__analyzers[name] = os.path.join(
                     self._data_files_dir_path, value)
-                # For packaged analyzers the ld_library path
-                # must be extended with the packed libs.
-                self.__analyzer_envs[name] =\
-                    env.extend(self.path_env_extra, self.ld_lib_path_extra)
             else:
                 env_path = cc_env['PATH'] if cc_env else None
                 compiler_binary = which(cmd=value, path=env_path)
@@ -224,17 +251,12 @@ def __populate_analyzers(self):
                     continue
 
                 self.__analyzers[name] = os.path.realpath(compiler_binary)
-                # For non-packaged analyzers the original env is set.
-                self.__analyzer_envs[name] = env.get_original_env()
 
                 # If the compiler binary is a simlink to ccache, use the
                 # original compiler binary.
                 if self.__analyzers[name].endswith("/ccache"):
                     self.__analyzers[name] = compiler_binary
 
-    def get_analyzer_env(self, analyzer_name):
-        return self.__analyzer_envs.get(analyzer_name)
-
     def __populate_replacer(self):
         """ Set clang-apply-replacements tool. """
         replacer_binary = self.pckg_layout.get('clang-apply-replacements')
@@ -243,12 +265,8 @@ def __populate_replacer(self):
             # Check if it is a package relative path.
             self.__replacer = os.path.join(self._data_files_dir_path,
                                            replacer_binary)
-            self.__analyzer_envs['clang-apply-replacements'] =\
-                env.extend(self.path_env_extra, self.ld_lib_path_extra)
         else:
             self.__replacer = which(replacer_binary)
-            self.__analyzer_envs['clang-apply-replacements'] =\
-                env.get_original_env()
 
     @property
     def version(self):

analyzer/codechecker_analyzer/analyzers/analyzer_base.py

@@ -58,7 +58,7 @@ def resolve_missing_binary(cls, configured_binary, environ):
 
     @classmethod
     @abstractmethod
-    def get_binary_version(cls, environ, details=False) -> str:
+    def get_binary_version(cls, details=False) -> str:
         """
         Return the version number of the binary that CodeChecker found, even
         if its incompatible. If details is true, additional version information
@@ -68,7 +68,7 @@ def get_binary_version(cls, environ, details=False) -> str:
         raise NotImplementedError("Subclasses should implement this!")
 
     @classmethod
-    def is_binary_version_incompatible(cls, environ) -> Optional[str]:
+    def is_binary_version_incompatible(cls) -> Optional[str]:
         """
         CodeChecker can only execute certain versions of analyzers.
         Returns a error object (an optional string). If the return value is
@@ -102,7 +102,7 @@ def construct_result_handler(self, buildaction, report_output,
         """
         raise NotImplementedError("Subclasses should implement this!")
 
-    def analyze(self, analyzer_cmd, res_handler, proc_callback=None, env=None):
+    def analyze(self, analyzer_cmd, res_handler, proc_callback=None):
         """
         Run the analyzer.
         """
@@ -111,17 +111,12 @@ def analyze(self, analyzer_cmd, res_handler, proc_callback=None, env=None):
         LOG.debug_analyzer('\n%s',
                            ' '.join([shlex.quote(x) for x in analyzer_cmd]))
 
-        if env is None:
-            env = analyzer_context.get_context()\
-                .get_analyzer_env(self.ANALYZER_NAME)
-
         res_handler.analyzer_cmd = analyzer_cmd
         try:
             ret_code, stdout, stderr \
                 = SourceAnalyzer.run_proc(analyzer_cmd,
                                           res_handler.buildaction.directory,
-                                          proc_callback,
-                                          env)
+                                          proc_callback)
             res_handler.analyzer_returncode = ret_code
             res_handler.analyzer_stdout = stdout
             res_handler.analyzer_stderr = stderr
@@ -145,7 +140,7 @@ def post_analyze(self, result_handler):
         """
 
     @staticmethod
-    def run_proc(command, cwd=None, proc_callback=None, env=None):
+    def run_proc(command, cwd=None, proc_callback=None):
         """
         Just run the given command and return the return code
         and the stdout and stderr outputs of the process.
@@ -161,9 +156,9 @@ def signal_handler(signum, _):
 
         signal.signal(signal.SIGINT, signal_handler)
 
-        if env is None:
-            env = analyzer_context.get_context().cc_env
+        env = analyzer_context.get_context().get_env_for_bin(command[0])
 
+        LOG.debug_analyzer('\nexecuting:%s\n', command)
         LOG.debug_analyzer('\nENV:\n')
         LOG.debug_analyzer(env)
 

analyzer/codechecker_analyzer/analyzers/analyzer_types.py

@@ -105,8 +105,8 @@ def is_ignore_conflict_supported():
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             env=context
-                            .get_analyzer_env(
-                                os.path.basename(context.replacer_binary)),
+                            .get_env_for_bin(
+                                context.replacer_binary),
                             encoding="utf-8", errors="ignore")
     out, _ = proc.communicate()
     return '--ignore-insert-conflict' in out
@@ -159,7 +159,6 @@ def check_supported_analyzers(analyzers):
     """
 
     context = analyzer_context.get_context()
-    check_env = context.cc_env
 
     analyzer_binaries = context.analyzer_binaries
 
@@ -182,7 +181,8 @@ def check_supported_analyzers(analyzers):
         elif not os.path.isabs(analyzer_bin):
             # If the analyzer is not in an absolute path, try to find it...
             found_bin = supported_analyzers[analyzer_name].\
-                resolve_missing_binary(analyzer_bin, check_env)
+                resolve_missing_binary(analyzer_bin,
+                                       context.get_env_for_bin(analyzer_bin))
 
             # found_bin is an absolute path, an executable in one of the
             # PATH folders.
@@ -201,7 +201,7 @@ def check_supported_analyzers(analyzers):
         # Check version compatibility of the analyzer binary.
         if analyzer_bin:
             analyzer = supported_analyzers[analyzer_name]
-            error = analyzer.is_binary_version_incompatible(check_env)
+            error = analyzer.is_binary_version_incompatible()
             if error:
                 failed_analyzers.add((analyzer_name,
                                       f"Incompatible version: {error} "
@@ -211,7 +211,7 @@ def check_supported_analyzers(analyzers):
                 available_analyzer = False
 
         if not analyzer_bin or \
-           not host_check.check_analyzer(analyzer_bin, check_env):
+           not host_check.check_analyzer(analyzer_bin):
             # Analyzers unavailable under absolute paths are deliberately a
             # configuration problem.
             failed_analyzers.add((analyzer_name,

analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py

@@ -52,7 +52,7 @@ def parse_clang_help_page(
             command,
             stderr=subprocess.STDOUT,
             env=analyzer_context.get_context()
-            .get_analyzer_env(ClangSA.ANALYZER_NAME),
+            .get_env_for_bin(command[0]),
             universal_newlines=True,
             encoding="utf-8",
             errors="ignore")
@@ -172,8 +172,11 @@ def __add_plugin_load_flags(cls, analyzer_cmd: List[str]):
             analyzer_cmd.extend(["-load", plugin])
 
     @classmethod
-    def get_binary_version(cls, environ, details=False) -> str:
+    def get_binary_version(cls, details=False) -> str:
         # No need to LOG here, we will emit a warning later anyway.
+
+        environ = analyzer_context.get_context().get_env_for_bin(
+            cls.analyzer_binary())
         if not cls.analyzer_binary():
             return None
 
@@ -209,7 +212,7 @@ def ctu_capability(cls):
             cls.__ctu_autodetection = CTUAutodetection(
                 cls.analyzer_binary(),
                 analyzer_context.get_context()
-                .get_analyzer_env(ClangSA.ANALYZER_NAME))
+                .get_env_for_bin(cls.analyzer_binary()))
 
         return cls.__ctu_autodetection
 
@@ -587,7 +590,7 @@ def resolve_missing_binary(cls, configured_binary, environ):
         return clang
 
     @classmethod
-    def is_binary_version_incompatible(cls, environ):
+    def is_binary_version_incompatible(cls):
         """
         We support pretty much every ClangSA version.
         """
@@ -609,7 +612,8 @@ def construct_result_handler(self, buildaction, report_output,
     def construct_config_handler(cls, args):
 
         context = analyzer_context.get_context()
-        environ = context.get_analyzer_env(ClangSA.ANALYZER_NAME)
+        environ = context.get_env_for_bin(
+            cls.analyzer_binary())
 
         handler = config_handler.ClangSAConfigHandler(environ)
 

analyzer/codechecker_analyzer/analyzers/clangsa/ctu_manager.py

@@ -126,11 +126,10 @@ def generate_ast(triple_arch, action, source, config):
             os.makedirs(ast_dir)
         except OSError:
             pass
-
     cmdstr = ' '.join(cmd)
     LOG.debug_analyzer("Generating AST using '%s'", cmdstr)
     ret_code, _, err = \
-        analyzer_base.SourceAnalyzer.run_proc(cmd, action.directory)
+        analyzer_base.SourceAnalyzer.run_proc(cmd, action.directory, None)
 
     if ret_code != 0:
         LOG.error("Error generating AST.\n\ncommand:\n\n%s\n\nstderr:\n\n%s",

analyzer/codechecker_analyzer/analyzers/clangsa/version.py

@@ -77,7 +77,7 @@ def get(clang_binary):
     """
     compiler_version = subprocess.check_output(
         [clang_binary, '--version'],
-        env=analyzer_context.get_context().get_analyzer_env("clangsa"),
+        env=analyzer_context.get_context().get_env_for_bin(clang_binary),
         encoding="utf-8",
         errors="ignore")
     version_parser = ClangVersionInfoParser(clang_binary)

analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py

@@ -236,8 +236,10 @@ def analyzer_binary(cls):
             .analyzer_binaries[cls.ANALYZER_NAME]
 
     @classmethod
-    def get_binary_version(cls, environ, details=False) -> str:
+    def get_binary_version(cls, details=False) -> str:
         # No need to LOG here, we will emit a warning later anyway.
+        environ = analyzer_context.get_context().get_env_for_bin(
+            cls.analyzer_binary())
         if not cls.analyzer_binary():
             return None
 
@@ -271,7 +273,7 @@ def get_analyzer_checkers(cls):
                 return cls.__analyzer_checkers
 
             environ = analyzer_context\
-                .get_context().get_analyzer_env(cls.ANALYZER_NAME)
+                .get_context().get_env_for_bin(cls.analyzer_binary())
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-list-checks", "-checks=*"],
                 env=environ,
@@ -299,7 +301,7 @@ def get_checker_config(cls):
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-dump-config", "-checks=*"],
                 env=analyzer_context.get_context()
-                .get_analyzer_env(cls.ANALYZER_NAME),
+                .get_env_for_bin(cls.analyzer_binary()),
                 universal_newlines=True,
                 encoding="utf-8",
                 errors="ignore")
@@ -316,7 +318,7 @@ def get_analyzer_config(cls):
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-dump-config", "-checks=*"],
                 env=analyzer_context.get_context()
-                .get_analyzer_env(cls.ANALYZER_NAME),
+                .get_env_for_bin(cls.analyzer_binary()),
                 universal_newlines=True,
                 encoding="utf-8",
                 errors="ignore")
@@ -567,7 +569,7 @@ def resolve_missing_binary(cls, configured_binary, environ):
         return clangtidy
 
     @classmethod
-    def is_binary_version_incompatible(cls, environ):
+    def is_binary_version_incompatible(cls):
         """
         We support pretty much every Clang-Tidy version.
         """

analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py

@@ -82,9 +82,11 @@ def analyzer_binary(cls):
             .analyzer_binaries[cls.ANALYZER_NAME]
 
     @classmethod
-    def get_binary_version(cls, environ, details=False) -> str:
+    def get_binary_version(cls, details=False) -> str:
         """ Get analyzer version information. """
         # No need to LOG here, we will emit a warning later anyway.
+        environ = analyzer_context.get_context().get_env_for_bin(
+            cls.analyzer_binary())
         if not cls.analyzer_binary():
             return None
         version = [cls.analyzer_binary(), '--version']
@@ -324,11 +326,11 @@ def resolve_missing_binary(cls, configured_binary, environ):
         return cppcheck
 
     @classmethod
-    def is_binary_version_incompatible(cls, environ):
+    def is_binary_version_incompatible(cls):
         """
         Check the version compatibility of the given analyzer binary.
         """
-        analyzer_version = cls.get_binary_version(environ)
+        analyzer_version = cls.get_binary_version()
 
         # The analyzer version should be above 1.80 because '--plist-output'
         # argument was introduced in this release.