From 0713a51c7f7fa9542edf9682390db78e0c77f547 Mon Sep 17 00:00:00 2001 From: Jiahui Xie Date: Fri, 1 Dec 2017 01:36:23 -0700 Subject: [PATCH 1/4] Implement 'Guess Include Path' feature in 'template.py' - Add 'docstring' to all functions in 'template.py' - Implement new functions 'GuessBuildDirectory' 'GuessIncludeDirectory', 'GuessSourceDirectory', and 'TraverseByDepth' in 'template.py' - Make all the code in 'template.py' PEP 8 conformant --- template.py | 365 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 279 insertions(+), 86 deletions(-) diff --git a/template.py b/template.py index 7f52bb4..65c3731 100644 --- a/template.py +++ b/template.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # This file is NOT licensed under the GPLv3, which is the license for the rest # of YouCompleteMe. # @@ -28,25 +30,50 @@ # # For more information, please refer to +''' +YouCompleteMe configuration file template to be used in 'YCM-Generator'. +''' + # Needed because ur"" syntax is no longer supported from __future__ import unicode_literals +import glob import os -import ycm_core import re import subprocess +import ycm_core +# ========================== Configuration Options ============================ +GUESS_BUILD_DIRECTORY = False # Experimental +GUESS_INCLUDE_PATH = True +# ========================== Configuration Options ============================ + +# =========================== Constant Definitions ============================ +SOURCE_EXTENSIONS = ('.C', '.cpp', '.cxx', '.cc', '.c', '.m', '.mm') +HEADER_EXTENSIONS = ('.H', '.h', '.hxx', '.hpp', '.hh') +# =========================== Constant Definitions ============================ + flags = [ # INSERT FLAGS HERE ] def LoadSystemIncludes(): - regex = re.compile(r'(?:\#include \<...\> search starts here\:)(?P.*?)(?:End of search list)', re.DOTALL) - process = subprocess.Popen(['clang', '-v', '-E', '-x', 'c++', '-'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ''' + Return a list of system include paths prefixed by '-isystem'. + ''' + query = ( + r'(?:\#include \<...\> search starts here\:)' + r'(?P.*?)(?:End of search list)' + ) + regex = re.compile(query, re.DOTALL) + process = subprocess.Popen(['clang', '-v', '-E', '-x', 'c++', '-'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) process_out, process_err = process.communicate('') - output = (process_out + process_err).decode("utf8") + output = (process_out + process_err).decode('utf8') includes = [] for p in re.search(regex, output).group('list').split('\n'): @@ -56,6 +83,147 @@ def LoadSystemIncludes(): includes.append(p) return includes + +def DirectoryOfThisScript(): + ''' + Return the absolute path of the parent directory containing this + script. + ''' + return os.path.dirname(os.path.abspath(__file__)) + + +def GuessBuildDirectory(): + ''' + Guess the build directory using the following heuristics: + + 1. Returns the current directory of this script plus 'build' + subdirectory in absolute path if this subdirectory exists. + + 2. Otherwise, probes whether there exists any directory + containing 'compile_commands.json' file two levels above the current + directory; returns this single directory only if there is one candidate. + + Raise 'OSError' if the two above fail. + ''' + result = os.path.join(DirectoryOfThisScript(), 'build') + + if os.path.exists(result): + return result + + result = glob.glob(os.path.join(DirectoryOfThisScript(), + '..', '..', '*', 'compile_commands.json')) + + if not result: + raise OSError('Build directory cannot be guessed.') + + if 1 != len(result): + raise OSError('Build directory cannot be guessed.') + + return os.path.split(result[0])[0] + + +def GuessIncludeDirectory(): + ''' + Return a list of absolute include paths; the list would be empty if the + attempt fails. + + NOTE: + 1. It first probes whether there are any paths in the result of + 'GuessSourceDirectory' containing files with extensions specified in + 'HEADER_EXTENSIONS', qualified paths would become part of the result. + + 2. It then checks the existence of either 'include' or 'inclues' in the + current directory; it will add them to the result if the directory exists + regardless of whether the directory contain valid file extensions + specified in 'HEADER_EXTENSIONS'. + ''' + result = list() + source_dir = None + include_dir_set = set() + external_include_dir_set = set() + + try: + source_dir = os.path.join(DirectoryOfThisScript(), + GuessSourceDirectory()) + include_dir_set = TraverseByDepth(source_dir, + frozenset(HEADER_EXTENSIONS)) + except OSError: + pass + + # The 'include' or 'includes' subdirectory can be left as empty, but still + # be considered as valid include path; unlike the include paths that reside + # inside source directory. + for external in ('include', 'includes'): + external_include_path = os.path.join(DirectoryOfThisScript(), external) + if os.path.exists(external_include_path): + external_include_dir_set = TraverseByDepth(external_include_path, + None) + break + + if include_dir_set: + for include_dir in include_dir_set: + result.append('-I' + include_dir) + + if external_include_dir_set: + for include_dir in external_include_dir_set: + result.append('-I' + include_dir) + + return result + + +def GuessSourceDirectory(): + ''' + Return either 'src', 'lib', or the name of the parent directory if any one + of them exists in the current directory; the first found result is returned. + Otherwise 'OSError' is raised should all of the above fail. + ''' + guess_candidates = ( + 'src', + 'lib', + os.path.basename(DirectoryOfThisScript()) + ) + + for candidate in guess_candidates: + result = os.path.join(DirectoryOfThisScript(), candidate) + + if os.path.exists(result): + return result + raise OSError('Source directory cannot be guessed.') + + +def TraverseByDepth(root, include_extensions): + ''' + Return a set of child directories of the 'root' containing file + extensions specified in 'include_extensions'. + + NOTE: + 1. The 'root' directory itself is excluded from the result set. + 2. No subdirectories would be excluded if 'include_extensions' is left + to 'None'. + 3. Each entry in 'include_extensions' must begin with string '.'. + ''' + is_root = True + result = set() + # Perform a depth first top down traverse of the given directory tree. + for root_dir, subdirs, file_list in os.walk(root): + if not is_root: + # print("Relative Root: ", root_dir) + # print(subdirs) + if include_extensions: + get_ext = os.path.splitext + subdir_extensions = { + get_ext(f)[-1] for f in file_list if get_ext(f)[-1] + } + if subdir_extensions & include_extensions: + result.add(root_dir) + else: + result.add(root_dir) + else: + is_root = False + + return result + + # Set this to the absolute path to the folder (NOT the file!) containing the # compile_commands.json file to use that instead of 'flags'. See here for # more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html @@ -68,89 +236,114 @@ def LoadSystemIncludes(): # 'flags' list of compilation flags. Notice that YCM itself uses that approach. compilation_database_folder = '' -if os.path.exists( compilation_database_folder ): - database = ycm_core.CompilationDatabase( compilation_database_folder ) +if GUESS_INCLUDE_PATH: + flags.extend(GuessIncludeDirectory()) + +if GUESS_BUILD_DIRECTORY: + try: + compilation_database_folder = GuessBuildDirectory() + except OSError: + compilation_database_folder = '' + +if os.path.exists(compilation_database_folder): + database = ycm_core.CompilationDatabase(compilation_database_folder) else: - database = None + database = None -SOURCE_EXTENSIONS = [ '.C', '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ] -systemIncludes = LoadSystemIncludes() -flags = flags + systemIncludes +system_includes = LoadSystemIncludes() +flags = flags + system_includes -def DirectoryOfThisScript(): - return os.path.dirname( os.path.abspath( __file__ ) ) - - -def MakeRelativePathsInFlagsAbsolute( flags, working_directory ): - if not working_directory: - return list( flags ) - new_flags = [] - make_next_absolute = False - path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ] - for flag in flags: - new_flag = flag - - if make_next_absolute: - make_next_absolute = False - if not flag.startswith( '/' ): - new_flag = os.path.join( working_directory, flag ) - - for path_flag in path_flags: - if flag == path_flag: - make_next_absolute = True - break - - if flag.startswith( path_flag ): - path = flag[ len( path_flag ): ] - new_flag = path_flag + os.path.join( working_directory, path ) - break - - if new_flag: - new_flags.append( new_flag ) - return new_flags - - -def IsHeaderFile( filename ): - extension = os.path.splitext( filename )[ 1 ] - return extension in [ '.H', '.h', '.hxx', '.hpp', '.hh' ] - - -def GetCompilationInfoForFile( filename ): - # The compilation_commands.json file generated by CMake does not have entries - # for header files. So we do our best by asking the db for flags for a - # corresponding source file, if any. If one exists, the flags for that file - # should be good enough. - if IsHeaderFile( filename ): - basename = os.path.splitext( filename )[ 0 ] - for extension in SOURCE_EXTENSIONS: - replacement_file = basename + extension - if os.path.exists( replacement_file ): - compilation_info = database.GetCompilationInfoForFile( - replacement_file ) - if compilation_info.compiler_flags_: - return compilation_info - return None - return database.GetCompilationInfoForFile( filename ) - - -def FlagsForFile( filename, **kwargs ): - if database: - # Bear in mind that compilation_info.compiler_flags_ does NOT return a - # python list, but a "list-like" StringVec object - compilation_info = GetCompilationInfoForFile( filename ) - if not compilation_info: - return None - - final_flags = MakeRelativePathsInFlagsAbsolute( - compilation_info.compiler_flags_, - compilation_info.compiler_working_dir_ ) + systemIncludes - - else: - relative_to = DirectoryOfThisScript() - final_flags = MakeRelativePathsInFlagsAbsolute( flags, relative_to ) - - return { - 'flags': final_flags, - 'do_cache': True - } +def MakeRelativePathsInFlagsAbsolute(flags, working_directory): + ''' + Iterate through 'flags' and replace the relative paths prefixed by + '-isystem', '-I', '-iquote', '--sysroot=' with absolute paths + start with 'working_directory'. + ''' + if not working_directory: + return list(flags) + new_flags = [] + make_next_absolute = False + path_flags = ['-isystem', '-I', '-iquote', '--sysroot='] + for flag in flags: + new_flag = flag + + if make_next_absolute: + make_next_absolute = False + if not flag.startswith('/'): + new_flag = os.path.join(working_directory, flag) + + for path_flag in path_flags: + if flag == path_flag: + make_next_absolute = True + break + + if flag.startswith(path_flag): + path = flag[len(path_flag):] + new_flag = path_flag + os.path.join(working_directory, path) + break + + if new_flag: + new_flags.append(new_flag) + return new_flags + + +def IsHeaderFile(filename): + ''' + Check whether 'filename' is considered as a header file. + ''' + extension = os.path.splitext(filename)[1] + return extension in HEADER_EXTENSIONS + + +def GetCompilationInfoForFile(filename): + ''' + Helper function to look up compilation info of 'filename' in the 'database'. + ''' + # The compilation_commands.json file generated by CMake does not have + # entries for header files. So we do our best by asking the db for flags for + # a corresponding source file, if any. If one exists, the flags for that + # file should be good enough. + if not database: + return None + + if IsHeaderFile(filename): + basename = os.path.splitext(filename)[0] + for extension in SOURCE_EXTENSIONS: + replacement_file = basename + extension + if os.path.exists(replacement_file): + compilation_info = \ + database.GetCompilationInfoForFile(replacement_file) + if compilation_info.compiler_flags_: + return compilation_info + return None + return database.GetCompilationInfoForFile(filename) + + +def FlagsForFile(filename, **kwargs): + ''' + Callback function to be invoked by YouCompleteMe in order to get the + information necessary to compile 'filename'. + + It returns a dictionary with a single element 'flags'. This element is a + list of compiler flags to pass to libclang for the file 'filename'. + ''' + if database: + # Bear in mind that compilation_info.compiler_flags_ does NOT return a + # python list, but a "list-like" StringVec object + compilation_info = GetCompilationInfoForFile(filename) + if not compilation_info: + return None + + final_flags = MakeRelativePathsInFlagsAbsolute( + compilation_info.compiler_flags_, + compilation_info.compiler_working_dir_) + system_includes + + else: + relative_to = DirectoryOfThisScript() + final_flags = MakeRelativePathsInFlagsAbsolute(flags, relative_to) + + return { + 'flags': final_flags, + 'do_cache': True + } From a2c00e97cda587805aad8d8849ac83da821ef64a Mon Sep 17 00:00:00 2001 From: Jiahui Xie Date: Fri, 1 Dec 2017 03:55:07 -0700 Subject: [PATCH 2/4] Add 'source' as an extra source tree guessing candidate - Add 'source' to the candidate list of 'GuessSourceDirectory' function of 'template.py' --- template.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/template.py b/template.py index 65c3731..c0bfbc5 100644 --- a/template.py +++ b/template.py @@ -173,12 +173,14 @@ def GuessIncludeDirectory(): def GuessSourceDirectory(): ''' - Return either 'src', 'lib', or the name of the parent directory if any one - of them exists in the current directory; the first found result is returned. + Return either 'src', 'source', 'lib', or the name of the parent directory if + any one of them exists in the current directory; the first found result is + returned. Otherwise 'OSError' is raised should all of the above fail. ''' guess_candidates = ( 'src', + 'source', 'lib', os.path.basename(DirectoryOfThisScript()) ) From 05fbf8d8000740bf9f8dd5c3c170b444f0f5b903 Mon Sep 17 00:00:00 2001 From: Jiahui Xie Date: Fri, 1 Dec 2017 04:17:39 -0700 Subject: [PATCH 3/4] Remove the interpreter directive in 'template.py' --- template.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/template.py b/template.py index c0bfbc5..7d86934 100644 --- a/template.py +++ b/template.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # This file is NOT licensed under the GPLv3, which is the license for the rest # of YouCompleteMe. # From 55ccffb537eca9d786517491ae00e42ab30e604d Mon Sep 17 00:00:00 2001 From: Jiahui Xie Date: Sun, 3 Dec 2017 06:40:41 -0700 Subject: [PATCH 4/4] Make the file extension comparison case-insensitive - Add extra in-source documentation for 'GUESS_BUILD_DIRECTORY' and 'GUESS_INCLUDE_PATH' - Modify 'TraverseByDepth' function to behave in a case-insensitive manner --- template.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/template.py b/template.py index 7d86934..fad5f0d 100644 --- a/template.py +++ b/template.py @@ -43,13 +43,25 @@ # ========================== Configuration Options ============================ -GUESS_BUILD_DIRECTORY = False # Experimental +# Refer to the docstring of 'GuessBuildDirectory' function for further detail. +# This is an experimental feature. +GUESS_BUILD_DIRECTORY = False +# Refer to the docstring of 'GuessIncludeDirectory' function for further detail. GUESS_INCLUDE_PATH = True # ========================== Configuration Options ============================ # =========================== Constant Definitions ============================ -SOURCE_EXTENSIONS = ('.C', '.cpp', '.cxx', '.cc', '.c', '.m', '.mm') -HEADER_EXTENSIONS = ('.H', '.h', '.hxx', '.hpp', '.hh') +# NOTE: +# +# 1. The string comparison in this configuration file is performed in a case +# in-sensitive manner; for example, there is no difference between file +# extensions of the following: '.cc' '.CC' '.Cc' 'cC' +# +# 2. One of the naming conventions for C++ header and source files involve the +# use of uppercase '.H' and '.C' - this case is handled as if they are named +# as '.h', and '.c', respectively. +SOURCE_EXTENSIONS = ('.cpp', '.cxx', '.cc', '.c', '.m', '.mm') +HEADER_EXTENSIONS = ('.h', '.hxx', '.hpp', '.hh') # =========================== Constant Definitions ============================ flags = [ @@ -201,9 +213,16 @@ def TraverseByDepth(root, include_extensions): 2. No subdirectories would be excluded if 'include_extensions' is left to 'None'. 3. Each entry in 'include_extensions' must begin with string '.'. + 4. Each entry in 'include_extensions' is treated in a case-insensitive + manner. ''' is_root = True result = set() + + if include_extensions: + new_extensions = { entry.lower() for entry in include_extensions } + include_extensions = new_extensions + # Perform a depth first top down traverse of the given directory tree. for root_dir, subdirs, file_list in os.walk(root): if not is_root: @@ -212,7 +231,7 @@ def TraverseByDepth(root, include_extensions): if include_extensions: get_ext = os.path.splitext subdir_extensions = { - get_ext(f)[-1] for f in file_list if get_ext(f)[-1] + get_ext(f)[-1].lower() for f in file_list if get_ext(f)[-1] } if subdir_extensions & include_extensions: result.add(root_dir) @@ -292,7 +311,7 @@ def IsHeaderFile(filename): ''' Check whether 'filename' is considered as a header file. ''' - extension = os.path.splitext(filename)[1] + extension = os.path.splitext(filename)[1].lower() return extension in HEADER_EXTENSIONS