ESP32 xtensa toolchain

.travis.yml

@@ -0,0 +1,15 @@
+language: python
+python:
+    - "2.7"
+    - "3.6"
+
+addons:
+    apt:
+        packages:
+            - clang
+
+script:
+    - python --version
+    - python config_gen.py --help
+    - env PYTHONPATH=mocks python template.py
+

README.md

@@ -1,4 +1,11 @@
 # YCM-Generator
+|                                                                                                                                  |                    |
+| -------------------------------------------------------------------------------------------------------------------------------- | ------------------ |
+| [![Build Status](https://travis-ci.org/rdnetto/YCM-Generator.svg?branch=stable)](https://travis-ci.org/rdnetto/YCM-Generator)    | Stable branch      |
+| [![Build Status](https://travis-ci.org/rdnetto/YCM-Generator.svg?branch=develop)](https://travis-ci.org/rdnetto/YCM-Generator)   | Development branch |
+
+## Introduction
+
 This is a script which generates a list of compiler flags from a project with an arbitrary build system. It can be used to:
 
 * generate a ```.ycm_extra_conf.py``` file for use with [YouCompleteMe](https://github.com/Valloric/YouCompleteMe)
@@ -13,7 +20,7 @@ Add ```NeoBundle 'rdnetto/YCM-Generator'``` to your vimrc (or the equivalent for
 
 For [vim-plug](https://github.com/junegunn/vim-plug) users, add ```Plug 'rdnetto/YCM-Generator', { 'branch': 'stable'}``` to your vimrc.
 
-Alternatively, Arch Linux users can install YCM-Generator using the (unofficial) [AUR package](https://aur4.archlinux.org/packages/ycm-generator-git/).
+Alternatively, Arch Linux users can install YCM-Generator using the (unofficial) [AUR package](https://aur.archlinux.org/packages/ycm-generator-git/).
 
 ## Usage
 Run ```./config_gen.py PROJECT_DIRECTORY```, where ```PROJECT_DIRECTORY``` is the root directory of your project's build system (i.e. the one containing the root Makefile, etc.)
@@ -30,6 +37,7 @@ You can also invoke it from within Vim using the ```:YcmGenerateConfig``` or ```
     + cmake
     + qmake
     + autotools
+    + meson (Ninja)
 
 Your build system should support specifying the compiler through the ```CC```/```CXX``` environment variables, or not use an absolute path to the compiler.
 
@@ -63,6 +71,7 @@ The following projects are used for testing:
 | [Clementine](https://github.com/clementine-player/Clementine.git)         | Cmake             |        |
 | [ExtPlane](https://github.com/vranki/ExtPlane.git)                        | Qmake             | Should be tested with both versions of Qt. |
 | [OpenFOAM](https://github.com/OpenFOAM/OpenFOAM-3.0.x.git)                | wmake             |        |
+| [Nautilus](https://git.gnome.org/browse/nautilus                          | meson (Ninja)     |        |
 
 ## License
 YCM-Generator is published under the GNU GPLv3.

config_gen.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 import sys
 import os
@@ -18,25 +18,34 @@
 # Default flags for make
 default_make_flags = ["-i", "-j" + str(multiprocessing.cpu_count())]
 
+# Default flags for ninja
+default_ninja_flags = ["-j" + str(multiprocessing.cpu_count())]
+
 # Set YCM-Generator directory
 # Always obtain the real path to the directory where 'config_gen.py' lives as,
 # in some cases, it will be a symlink placed in '/usr/bin' (as is the case
 # with the Arch Linux AUR package) and it won't
 # be able to find the plugin directory.
 ycm_generator_dir = os.path.dirname(os.path.realpath(__file__))
 
+def isString(string):
+    if sys.version_info[0] == 2:
+        return isinstance(string, basestring)
+    elif sys.version_info[0] == 3:
+        return isinstance(string, str)
 
 def main():
     # parse command-line args
     parser = argparse.ArgumentParser(description="Automatically generates config files for YouCompleteMe")
     parser.add_argument("-v", "--verbose", action="store_true", help="Show output from build process")
     parser.add_argument("-f", "--force", action="store_true", help="Overwrite the file if it exists.")
     parser.add_argument("-m", "--make", default="make", help="Use the specified executable for make.")
-    parser.add_argument("-b", "--build-system", choices=["cmake", "autotools", "qmake", "make"], help="Force use of the specified build system rather than trying to autodetect.")
+    parser.add_argument("-b", "--build-system", choices=["meson", "cmake", "autotools", "qmake", "make"], help="Force use of the specified build system rather than trying to autodetect.")
     parser.add_argument("-c", "--compiler", help="Use the specified executable for clang. It should be the same version as the libclang used by YCM. The executable for clang++ will be inferred from this.")
     parser.add_argument("-C", "--configure_opts", default="", help="Additional flags to pass to configure/cmake/etc. e.g. --configure_opts=\"--enable-FEATURE\"")
     parser.add_argument("-F", "--format", choices=["ycm", "cc"], default="ycm", help="Format of output file (YouCompleteMe or color_coded). Default: ycm")
     parser.add_argument("-M", "--make-flags", help="Flags to pass to make when fake-building. Default: -M=\"{}\"".format(" ".join(default_make_flags)))
+    parser.add_argument("-N", "--ninja-flags", help="Flags to pass to ninja when fake-building. Default: -N=\"{}\"".format(" ".join (default_ninja_flags)))
     parser.add_argument("-o", "--output", help="Save the config file as OUTPUT. Default: .ycm_extra_conf.py, or .color_coded if --format=cc.")
     parser.add_argument("-x", "--language", choices=["c", "c++"], help="Only output flags for the given language. This defaults to whichever language has its compiler invoked the most.")
     parser.add_argument("--out-of-tree", action="store_true", help="Build autotools projects out-of-tree. This is a no-op for other project types.")
@@ -87,8 +96,10 @@ def main():
 
     # command-line args to pass to fake_build() using kwargs
     args["make_cmd"] = args.pop("make")
+    args["ninja_cmd"] = "ninja"
     args["configure_opts"] = shlex.split(args["configure_opts"])
     args["make_flags"] = default_make_flags if args["make_flags"] is None else shlex.split(args["make_flags"])
+    args["ninja_flags"] = default_ninja_flags if args["ninja_flags"] is None else shlex.split(args["ninja_flags"])
     force_lang = args.pop("language")
     output_format = args.pop("format")
     del args["compiler"]
@@ -102,8 +113,8 @@ def main():
     }[output_format]
 
     # temporary files to hold build logs
-    with tempfile.NamedTemporaryFile(mode="rw") as c_build_log:
-        with tempfile.NamedTemporaryFile(mode="rw") as cxx_build_log:
+    with tempfile.NamedTemporaryFile(mode="r+") as c_build_log:
+        with tempfile.NamedTemporaryFile(mode="r+") as cxx_build_log:
             # perform the actual compilation of flags
             fake_build(project_dir, c_build_log.name, cxx_build_log.name, **args)
             (c_count, c_skip, c_flags) = parse_flags(c_build_log)
@@ -140,7 +151,7 @@ def main():
             print("Created {} config file with {} {} flags".format(output_format.upper(), len(flags), lang.upper()))
 
 
-def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_cmd, build_system, cc, cxx, out_of_tree, configure_opts, make_flags, preserve_environment, qt_version):
+def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_cmd, ninja_cmd, build_system, cc, cxx, out_of_tree, configure_opts, make_flags, ninja_flags, preserve_environment, qt_version):
     '''Builds the project using the fake toolchain, to collect the compiler flags.
 
     project_dir: the directory containing the source files
@@ -154,6 +165,7 @@ def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_
     make_flags: additional flags for make
     preserve_environment: pass environment variables to build processes
     qt_version: The Qt version to use when building with qmake.
+    ninja_cmd: The ninja command to use.
     '''
 
     # TODO: add Windows support
@@ -175,7 +187,7 @@ def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_
     else:
         # Preserve HOME, since Cmake needs it to find some packages and it's
         # normally there anyway. See #26.
-        env = dict(map(lambda x: (x, os.environ[x]), ["HOME"]))
+        env = { x: os.environ[x] for x in ["HOME"] }
 
     env["PATH"]  = "{}:{}".format(fake_path, os.environ["PATH"])
     env["CC"] = "clang"
@@ -192,6 +204,8 @@ def fake_build(project_dir, c_build_log_path, cxx_build_log_path, verbose, make_
     # depend upon the existence of various output files
     make_args = [make_cmd] + make_flags
 
+    ninja_args = [ninja_cmd] + ninja_flags
+
     # Used for the qmake build system below
     pro_files = glob.glob(os.path.join(project_dir, "*.pro"))
 
@@ -210,6 +224,8 @@ def run(cmd, *args, **kwargs):
             build_system = "autotools"
         elif pro_files:
             build_system = "qmake"
+        elif os.path.exists(os.path.join(project_dir, "meson.build")):
+            build_system = "meson"
         elif any([os.path.exists(os.path.join(project_dir, x)) for x in ["GNUmakefile", "makefile", "Makefile"]]):
             build_system = "make"
 
@@ -297,13 +313,33 @@ def run(cmd, *args, **kwargs):
         print("Running qmake in '{}' with Qt {}...".format(build_dir, qt_version))
         run(["qmake"] + configure_opts + [pro_files[0]], env=env_config,
             **proc_opts)
+        if os.uname()[0] == "Darwin":
+            print("Patching generated makefile...")
+            patch_qmake_makefile(os.path.join(build_dir, "Makefile"))
 
         print("\nRunning make...")
         run(make_args, env=env, **proc_opts)
 
         print("\nCleaning up...")
         print("")
         shutil.rmtree(build_dir)
+
+    elif build_system == "meson":
+        # meson build system
+        build_dir = tempfile.mkdtemp()
+        proc_opts["cwd"] = build_dir
+        print("Configuring meson in '{}'...".format(build_dir))
+
+        print("\nRunning meson...",project_dir)
+        run(["meson", project_dir], env=env_config, **proc_opts)
+
+        print("\nRunning ninja...")
+        print(project_dir)
+        run(ninja_args, env=env, **proc_opts)
+
+        print("\nCleaning up...")
+        print("")
+        shutil.rmtree(build_dir)
 
     elif build_system == "make":
         # make
@@ -338,6 +374,33 @@ def run(cmd, *args, **kwargs):
     print("")
 
 
+def patch_qmake_makefile(makefile_path):
+    '''Function that patches Makefiles generated by qmake to set CC, CXX and LINK variables correctly on Mac OSX
+
+    makefile_path: path to the qmake generated Makefile; patched result is written to the same file
+    '''
+
+    variables = {
+        "CC": "clang",
+        "CXX": "clang++",
+        "LINK": "clang++"
+    }
+
+    def sub_func(match_obj):
+        variable = match_obj.group(1).strip()
+        if variable in variables:
+            new_value = variables[variable]
+            return "{}= {}".format(match_obj.group(1), new_value)
+        else:
+            return match_obj.group(0)
+
+    with open(makefile_path, "r") as f:
+        makefile_input = f.read()
+    makefile_output = re.sub(r"(\s*\w+\s*)=.*", sub_func, makefile_input)
+    with open(makefile_path, "w") as f:
+        f.write(makefile_output)
+
+
 def parse_flags(build_log):
     '''Creates a list of compiler flags from the build log.
 
@@ -401,7 +464,7 @@ def parse_flags(build_log):
     # Only specify one word size (the largest)
     # (Different sizes are used for different files in the linux kernel.)
     mRegex = re.compile("^-m[0-9]+$")
-    word_flags = list([f for f in flags if isinstance(f, basestring) and mRegex.match(f)])
+    word_flags = list([f for f in flags if isString(f) and mRegex.match(f)])
 
     if(len(word_flags) > 1):
         for flag in word_flags:
@@ -410,14 +473,14 @@ def parse_flags(build_log):
         flags.add(max(word_flags))
 
     # Resolve duplicate macro definitions (always choose the last value for consistency)
-    for name, values in define_flags.iteritems():
+    for name, values in define_flags.items():
         if(len(values) > 1):
             print("WARNING: {} distinct definitions of macro {} found".format(len(values), name))
             values.sort()
 
         flags.add("-D{}={}".format(name, values[0]))
 
-    return (line_count, skip_count, sorted(flags))
+    return (line_count, skip_count, sorted(flags, key=lambda x:x[0] if isinstance(x, tuple) else x))
 
 
 def generate_cc_conf(flags, config_file):
@@ -428,13 +491,12 @@ def generate_cc_conf(flags, config_file):
 
     with open(config_file, "w") as output:
         for flag in flags:
-            if(isinstance(flag, basestring)):
+            if(isString(flag)):
                 output.write(flag + "\n")
             else: # is tuple
                 for f in flag:
                     output.write(f + "\n")
 
-
 def generate_ycm_conf(flags, config_file):
     '''Generates the .ycm_extra_conf.py.
 
@@ -451,7 +513,7 @@ def generate_ycm_conf(flags, config_file):
                 if(line == "    # INSERT FLAGS HERE\n"):
                     # insert generated code
                     for flag in flags:
-                        if(isinstance(flag, basestring)):
+                        if(isString(flag)):
                             output.write("    '{}',\n".format(flag))
                         else: # is tuple
                             output.write("    '{}', '{}',\n".format(*flag))

fake-toolchain/Unix/arm-none-eabi-g++

@@ -0,0 +1 @@
+cxx

fake-toolchain/Unix/arm-none-eabi-gcc

@@ -0,0 +1 @@
+cc

fake-toolchain/Unix/avr-g++

@@ -0,0 +1 @@
+cxx

fake-toolchain/Unix/avr-gcc

@@ -0,0 +1 @@
+cc

mocks/ycm_core.py

@@ -0,0 +1,6 @@
+# This file defines a mock of the ycm_core module so we can test template.py by
+# running it.
+
+def CompilationDatabase(x):
+    pass
+

template.py

@@ -28,14 +28,34 @@
 #
 # For more information, please refer to <http://unlicense.org/>
 
+# Needed because ur"" syntax is no longer supported
+from __future__ import unicode_literals
+
 import os
 import ycm_core
+import re
+import subprocess
+
 
 flags = [
     # INSERT FLAGS HERE
 ]
 
 
+def LoadSystemIncludes():
+    regex = re.compile(r'(?:\#include \<...\> search starts here\:)(?P<list>.*?)(?:End of search list)', re.DOTALL)
+    process = subprocess.Popen(['clang', '-v', '-E', '-x', 'c++', '-'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    process_out, process_err = process.communicate('')
+    output = (process_out + process_err).decode("utf8")
+
+    includes = []
+    for p in re.search(regex, output).group('list').split('\n'):
+        p = p.strip()
+        if len(p) > 0 and p.find('(framework directory)') < 0:
+            includes.append('-isystem')
+            includes.append(p)
+    return includes
+
 # Set this to the absolute path to the folder (NOT the file!) containing the
 # compile_commands.json file to use that instead of 'flags'. See here for
 # more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html
@@ -54,6 +74,8 @@
   database = None
 
 SOURCE_EXTENSIONS = [ '.C', '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ]
+systemIncludes = LoadSystemIncludes()
+flags = flags + systemIncludes
 
 def DirectoryOfThisScript():
   return os.path.dirname( os.path.abspath( __file__ ) )
@@ -121,7 +143,7 @@ def FlagsForFile( filename, **kwargs ):
 
     final_flags = MakeRelativePathsInFlagsAbsolute(
       compilation_info.compiler_flags_,
-      compilation_info.compiler_working_dir_ )
+      compilation_info.compiler_working_dir_ ) + systemIncludes
 
   else:
     relative_to = DirectoryOfThisScript()