diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9a4e73c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +language: python +python: + - "2.7" + - "3.6" + +addons: + apt: + packages: + - clang + +script: + - python --version + - python config_gen.py --help + - env PYTHONPATH=mocks python template.py + diff --git a/README.md b/README.md index c45a787..481342b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ # YCM-Generator +| | | +| -------------------------------------------------------------------------------------------------------------------------------- | ------------------ | +| [![Build Status](https://travis-ci.org/rdnetto/YCM-Generator.svg?branch=stable)](https://travis-ci.org/rdnetto/YCM-Generator) | Stable branch | +| [![Build Status](https://travis-ci.org/rdnetto/YCM-Generator.svg?branch=develop)](https://travis-ci.org/rdnetto/YCM-Generator) | Development branch | + +## Introduction + This is a script which generates a list of compiler flags from a project with an arbitrary build system. It can be used to: * generate a ```.ycm_extra_conf.py``` file for use with [YouCompleteMe](https://github.com/Valloric/YouCompleteMe) @@ -13,7 +20,7 @@ Add ```NeoBundle 'rdnetto/YCM-Generator'``` to your vimrc (or the equivalent for For [vim-plug](https://github.com/junegunn/vim-plug) users, add ```Plug 'rdnetto/YCM-Generator', { 'branch': 'stable'}``` to your vimrc. -Alternatively, Arch Linux users can install YCM-Generator using the (unofficial) [AUR package](https://aur4.archlinux.org/packages/ycm-generator-git/). +Alternatively, Arch Linux users can install YCM-Generator using the (unofficial) [AUR package](https://aur.archlinux.org/packages/ycm-generator-git/). ## Usage Run ```./config_gen.py PROJECT_DIRECTORY```, where ```PROJECT_DIRECTORY``` is the root directory of your project's build system (i.e. the one containing the root Makefile, etc.) @@ -30,6 +37,7 @@ You can also invoke it from within Vim using the ```:YcmGenerateConfig``` or ``` + cmake + qmake + autotools + + meson (Ninja) Your build system should support specifying the compiler through the ```CC```/```CXX``` environment variables, or not use an absolute path to the compiler. @@ -63,6 +71,7 @@ The following projects are used for testing: | [Clementine](https://github.com/clementine-player/Clementine.git) | Cmake | | | [ExtPlane](https://github.com/vranki/ExtPlane.git) | Qmake | Should be tested with both versions of Qt. | | [OpenFOAM](https://github.com/OpenFOAM/OpenFOAM-3.0.x.git) | wmake | | +| [Nautilus](https://git.gnome.org/browse/nautilus | meson (Ninja) | | ## License YCM-Generator is published under the GNU GPLv3. diff --git a/config_gen.py b/config_gen.py index 54e89f9..6089b20 100755 --- a/config_gen.py +++ b/config_gen.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python import sys import os @@ -18,6 +18,9 @@ # Default flags for make default_make_flags = ["-i", "-j" + str(multiprocessing.cpu_count())] +# Default flags for ninja +default_ninja_flags = ["-j" + str(multiprocessing.cpu_count())] + # Set YCM-Generator directory # Always obtain the real path to the directory where 'config_gen.py' lives as, # in some cases, it will be a symlink placed in '/usr/bin' (as is the case @@ -25,6 +28,11 @@ # be able to find the plugin directory. ycm_generator_dir = os.path.dirname(os.path.realpath(__file__)) +def isString(string): + if sys.version_info[0] == 2: + return isinstance(string, basestring) + elif sys.version_info[0] == 3: + return isinstance(string, str) def main(): # parse command-line args @@ -32,11 +40,12 @@ def main(): parser.add_argument("-v", "--verbose", action="store_true", help="Show output from build process") parser.add_argument("-f", "--force", action="store_true", help="Overwrite the file if it exists.") parser.add_argument("-m", "--make", default="make", help="Use the specified executable for make.") - parser.add_argument("-b", "--build-system", choices=["cmake", "autotools", "qmake", "make"], help="Force use of the specified build system rather than trying to autodetect.") + parser.add_argument("-b", "--build-system", choices=["meson", "cmake", "autotools", "qmake", "make"], help="Force use of the specified build system rather than trying to autodetect.") parser.add_argument("-c", "--compiler", help="Use the specified executable for clang. It should be the same version as the libclang used by YCM. The executable for clang++ will be inferred from this.") parser.add_argument("-C", "--configure_opts", default="", help="Additional flags to pass to configure/cmake/etc. e.g. --configure_opts=\"--enable-FEATURE\"") parser.add_argument("-F", "--format", choices=["ycm", "cc"], default="ycm", help="Format of output file (YouCompleteMe or color_coded). Default: ycm") parser.add_argument("-M", "--make-flags", help="Flags to pass to make when fake-building. Default: -M=\"{}\"".format(" ".join(default_make_flags))) + parser.add_argument("-N", "--ninja-flags", help="Flags to pass to ninja when fake-building. Default: -N=\"{}\"".format(" ".join (default_ninja_flags))) parser.add_argument("-o", "--output", help="Save the config file as OUTPUT. Default: .ycm_extra_conf.py, or .color_coded if --format=cc.") parser.add_argument("-x", "--language", choices=["c", "c++"], help="Only output flags for the given language. This defaults to whichever language has its compiler invoked the most.") parser.add_argument("--out-of-tree", action="store_true", help="Build autotools projects out-of-tree. This is a no-op for other project types.") @@ -87,8 +96,10 @@ def main(): # command-line args to pass to fake_build() using kwargs args["make_cmd"] = args.pop("make") + args["ninja_cmd"] = "ninja" args["configure_opts"] = shlex.split(args["configure_opts"]) args["make_flags"] = default_make_flags if args["make_flags"] is None else shlex.split(args["make_flags"]) + args["ninja_flags"] = default_ninja_flags if args["ninja_flags"] is None else shlex.split(args["ninja_flags"]) force_lang = args.pop("language") output_format = args.pop("format") del args["compiler"] @@ -102,8 +113,8 @@ def main(): }[output_format] # temporary files to hold build logs - with tempfile.NamedTemporaryFile(mode="rw") as c_build_log: - with tempfile.NamedTemporaryFile(mode="rw") as cxx_build_log: + with tempfile.NamedTemporaryFile(mode="r+") as c_build_log: + with tempfile.NamedTemporaryFile(mode="r+") as cxx_build_log: # perform the actual compilation of flags fake_build(project_dir, c_build_log.name, cxx_build_log.name, **args) (c_count, c_skip, c_flags) = parse_flags(c_build_log) @@ -140,7 +151,7 @@ def main(): print("Created {} config file with {} {} flags".format(output_format.upper(), len(flags), lang.upper())) -def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_cmd, build_system, cc, cxx, out_of_tree, configure_opts, make_flags, preserve_environment, qt_version): +def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_cmd, ninja_cmd, build_system, cc, cxx, out_of_tree, configure_opts, make_flags, ninja_flags, preserve_environment, qt_version): '''Builds the project using the fake toolchain, to collect the compiler flags. project_dir: the directory containing the source files @@ -154,6 +165,7 @@ def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_ make_flags: additional flags for make preserve_environment: pass environment variables to build processes qt_version: The Qt version to use when building with qmake. + ninja_cmd: The ninja command to use. ''' # TODO: add Windows support @@ -175,7 +187,7 @@ def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_ else: # Preserve HOME, since Cmake needs it to find some packages and it's # normally there anyway. See #26. - env = dict(map(lambda x: (x, os.environ[x]), ["HOME"])) + env = { x: os.environ[x] for x in ["HOME"] } env["PATH"] = "{}:{}".format(fake_path, os.environ["PATH"]) env["CC"] = "clang" @@ -192,6 +204,8 @@ def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_ # depend upon the existence of various output files make_args = [make_cmd] + make_flags + ninja_args = [ninja_cmd] + ninja_flags + # Used for the qmake build system below pro_files = glob.glob(os.path.join(project_dir, "*.pro")) @@ -210,6 +224,8 @@ def run(cmd, *args, **kwargs): build_system = "autotools" elif pro_files: build_system = "qmake" + elif os.path.exists(os.path.join(project_dir, "meson.build")): + build_system = "meson" elif any([os.path.exists(os.path.join(project_dir, x)) for x in ["GNUmakefile", "makefile", "Makefile"]]): build_system = "make" @@ -297,6 +313,9 @@ def run(cmd, *args, **kwargs): print("Running qmake in '{}' with Qt {}...".format(build_dir, qt_version)) run(["qmake"] + configure_opts + [pro_files[0]], env=env_config, **proc_opts) + if os.uname()[0] == "Darwin": + print("Patching generated makefile...") + patch_qmake_makefile(os.path.join(build_dir, "Makefile")) print("\nRunning make...") run(make_args, env=env, **proc_opts) @@ -304,6 +323,23 @@ def run(cmd, *args, **kwargs): print("\nCleaning up...") print("") shutil.rmtree(build_dir) + + elif build_system == "meson": + # meson build system + build_dir = tempfile.mkdtemp() + proc_opts["cwd"] = build_dir + print("Configuring meson in '{}'...".format(build_dir)) + + print("\nRunning meson...",project_dir) + run(["meson", project_dir], env=env_config, **proc_opts) + + print("\nRunning ninja...") + print(project_dir) + run(ninja_args, env=env, **proc_opts) + + print("\nCleaning up...") + print("") + shutil.rmtree(build_dir) elif build_system == "make": # make @@ -338,6 +374,33 @@ def run(cmd, *args, **kwargs): print("") +def patch_qmake_makefile(makefile_path): + '''Function that patches Makefiles generated by qmake to set CC, CXX and LINK variables correctly on Mac OSX + + makefile_path: path to the qmake generated Makefile; patched result is written to the same file + ''' + + variables = { + "CC": "clang", + "CXX": "clang++", + "LINK": "clang++" + } + + def sub_func(match_obj): + variable = match_obj.group(1).strip() + if variable in variables: + new_value = variables[variable] + return "{}= {}".format(match_obj.group(1), new_value) + else: + return match_obj.group(0) + + with open(makefile_path, "r") as f: + makefile_input = f.read() + makefile_output = re.sub(r"(\s*\w+\s*)=.*", sub_func, makefile_input) + with open(makefile_path, "w") as f: + f.write(makefile_output) + + def parse_flags(build_log): '''Creates a list of compiler flags from the build log. @@ -401,7 +464,7 @@ def parse_flags(build_log): # Only specify one word size (the largest) # (Different sizes are used for different files in the linux kernel.) mRegex = re.compile("^-m[0-9]+$") - word_flags = list([f for f in flags if isinstance(f, basestring) and mRegex.match(f)]) + word_flags = list([f for f in flags if isString(f) and mRegex.match(f)]) if(len(word_flags) > 1): for flag in word_flags: @@ -410,14 +473,14 @@ def parse_flags(build_log): flags.add(max(word_flags)) # Resolve duplicate macro definitions (always choose the last value for consistency) - for name, values in define_flags.iteritems(): + for name, values in define_flags.items(): if(len(values) > 1): print("WARNING: {} distinct definitions of macro {} found".format(len(values), name)) values.sort() flags.add("-D{}={}".format(name, values[0])) - return (line_count, skip_count, sorted(flags)) + return (line_count, skip_count, sorted(flags, key=lambda x:x[0] if isinstance(x, tuple) else x)) def generate_cc_conf(flags, config_file): @@ -428,13 +491,12 @@ def generate_cc_conf(flags, config_file): with open(config_file, "w") as output: for flag in flags: - if(isinstance(flag, basestring)): + if(isString(flag)): output.write(flag + "\n") else: # is tuple for f in flag: output.write(f + "\n") - def generate_ycm_conf(flags, config_file): '''Generates the .ycm_extra_conf.py. @@ -451,7 +513,7 @@ def generate_ycm_conf(flags, config_file): if(line == " # INSERT FLAGS HERE\n"): # insert generated code for flag in flags: - if(isinstance(flag, basestring)): + if(isString(flag)): output.write(" '{}',\n".format(flag)) else: # is tuple output.write(" '{}', '{}',\n".format(*flag)) diff --git a/fake-toolchain/Unix/arm-none-eabi-g++ b/fake-toolchain/Unix/arm-none-eabi-g++ new file mode 120000 index 0000000..34836c6 --- /dev/null +++ b/fake-toolchain/Unix/arm-none-eabi-g++ @@ -0,0 +1 @@ +cxx \ No newline at end of file diff --git a/fake-toolchain/Unix/arm-none-eabi-gcc b/fake-toolchain/Unix/arm-none-eabi-gcc new file mode 120000 index 0000000..2652f5f --- /dev/null +++ b/fake-toolchain/Unix/arm-none-eabi-gcc @@ -0,0 +1 @@ +cc \ No newline at end of file diff --git a/fake-toolchain/Unix/avr-g++ b/fake-toolchain/Unix/avr-g++ new file mode 120000 index 0000000..34836c6 --- /dev/null +++ b/fake-toolchain/Unix/avr-g++ @@ -0,0 +1 @@ +cxx \ No newline at end of file diff --git a/fake-toolchain/Unix/avr-gcc b/fake-toolchain/Unix/avr-gcc new file mode 120000 index 0000000..2652f5f --- /dev/null +++ b/fake-toolchain/Unix/avr-gcc @@ -0,0 +1 @@ +cc \ No newline at end of file diff --git a/mocks/ycm_core.py b/mocks/ycm_core.py new file mode 100644 index 0000000..fcc5cbb --- /dev/null +++ b/mocks/ycm_core.py @@ -0,0 +1,6 @@ +# This file defines a mock of the ycm_core module so we can test template.py by +# running it. + +def CompilationDatabase(x): + pass + diff --git a/template.py b/template.py index 2985a70..7f52bb4 100644 --- a/template.py +++ b/template.py @@ -28,14 +28,34 @@ # # For more information, please refer to +# Needed because ur"" syntax is no longer supported +from __future__ import unicode_literals + import os import ycm_core +import re +import subprocess + flags = [ # INSERT FLAGS HERE ] +def LoadSystemIncludes(): + regex = re.compile(r'(?:\#include \<...\> search starts here\:)(?P.*?)(?:End of search list)', re.DOTALL) + process = subprocess.Popen(['clang', '-v', '-E', '-x', 'c++', '-'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process_out, process_err = process.communicate('') + output = (process_out + process_err).decode("utf8") + + includes = [] + for p in re.search(regex, output).group('list').split('\n'): + p = p.strip() + if len(p) > 0 and p.find('(framework directory)') < 0: + includes.append('-isystem') + includes.append(p) + return includes + # Set this to the absolute path to the folder (NOT the file!) containing the # compile_commands.json file to use that instead of 'flags'. See here for # more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html @@ -54,6 +74,8 @@ database = None SOURCE_EXTENSIONS = [ '.C', '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ] +systemIncludes = LoadSystemIncludes() +flags = flags + systemIncludes def DirectoryOfThisScript(): return os.path.dirname( os.path.abspath( __file__ ) ) @@ -121,7 +143,7 @@ def FlagsForFile( filename, **kwargs ): final_flags = MakeRelativePathsInFlagsAbsolute( compilation_info.compiler_flags_, - compilation_info.compiler_working_dir_ ) + compilation_info.compiler_working_dir_ ) + systemIncludes else: relative_to = DirectoryOfThisScript()