diff --git a/.gitignore b/.gitignore index c0fb662..6ad8696 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __pycache__/ venv/ dist/ *.egg-info/ +*.ini +*.xml \ No newline at end of file diff --git a/README.md b/README.md index b2b1127..362b376 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ After a process of generating a compile_commands.json, you can run processcdb wi processcdb --tool clang-tidy -This will try to locate the json file from current working directory and runs the tool, in this case -clang-tidy, against all files that are compiled and not blacklisted in processcdb comfig file or in +This will try to locate the json file from current working directory and runs the tool, in this case the +clang-tidy tool as defined in the config file, against all files that are compiled and not blacklisted in processcdb config file or in tools own configuration file and generates the output to standard output. If you need to run the tool when you don't have access to change the current working directory, you can pass `--cdb` and absolute location: @@ -77,7 +77,7 @@ capture the standard output or provide `--config` parameter. ## Configuration file Each tool has a separate section and each section can be configured either in the tool specific section or -in default. Minimal. single tool configuratio would look something like this: +in default. The minimal single tool configuration would look something like this: [clang-tidy] binary=C:\llvm-11.0.0\bin\clang-tidy.exe diff --git a/run_processcdb.bat b/run_processcdb.bat new file mode 100644 index 0000000..3981a45 --- /dev/null +++ b/run_processcdb.bat @@ -0,0 +1,6 @@ +@echo off +pushd + +D:\fone\branch\external\ego\dev\python3\bin\python.exe src/processcdb --tool clang-tidy --xml --output results.xml --cdb D:\fone\branch\external\ego\dev\clang_tools\compile_commands.json + +popd \ No newline at end of file diff --git a/src/processcdb/__init__.py b/src/processcdb/__init__.py index facf1ee..9db673b 100644 --- a/src/processcdb/__init__.py +++ b/src/processcdb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ._version import get_versions +from _version import get_versions __version__ = get_versions()["version"] del get_versions diff --git a/src/processcdb/__main__.py b/src/processcdb/__main__.py index ce782ed..b5bce86 100755 --- a/src/processcdb/__main__.py +++ b/src/processcdb/__main__.py @@ -3,11 +3,10 @@ import sys import json import traceback -from .cdb_tools import TOOLS -from .misc import remove_untouched_files, remove_dupes, argument_parser +from cdb_tools import TOOLS +from misc import remove_dupes, argument_parser from configparser import ConfigParser -from .logger import LOGGER as log, LOG_LEVELS - +from logger import LOGGER as log, LOG_LEVELS def main(): args = argument_parser(TOOLS).parse_args() @@ -39,8 +38,8 @@ def main(): if args.cdb.is_file(): cdb = json.loads(args.cdb.read_text()) if cdb: - if args.commit_a is not None: - cdb = remove_untouched_files(cdb, (args.commit_a, args.commit_b)) + #if args.commit_a is not None: + #cdb = filterByChangelist(cdb, (args.commit_a, args.commit_b)) if not args.allow_dupes: cdb = remove_dupes(cdb) diff --git a/src/processcdb/cdb_tools.py b/src/processcdb/cdb_tools.py index 8387cf4..5813391 100644 --- a/src/processcdb/cdb_tools.py +++ b/src/processcdb/cdb_tools.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -from .toolbase import Tool # noqa: F401 -from .clangtidy import ClangTidy # noqa: F401 -from .clang import Clang # noqa: F401 -from .lizard import Lizard # noqa: F401 -from .cppcheck import CppCheck # noqa: F401 +from toolbase import Tool # noqa: F401 +from clangtidy import ClangTidy # noqa: F401 +from clang import Clang # noqa: F401 +from lizard import Lizard # noqa: F401 +from cppcheck import CppCheck # noqa: F401 TOOLS = { "clang-tidy": ClangTidy, diff --git a/src/processcdb/clang.py b/src/processcdb/clang.py index 78266d1..7c041fd 100644 --- a/src/processcdb/clang.py +++ b/src/processcdb/clang.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from .misc import is_any_windows -from .toolbase import Tool +from misc import is_any_windows +from toolbase import Tool class Clang(Tool): @@ -16,8 +16,10 @@ def execute(self, cdb, args): if args.output is not None: arguments.extend([f"--output {args.output}"]) + absoluteBinaryPath = Path(self.binary).resolve() + if is_any_windows(): - arguments.extend([f"--use-analyzer {self.binary}"]) + arguments.extend([f"--use-analyzer {absoluteBinaryPath}"]) - final_command = f"{self.binary} {' '.join(arguments)}" + final_command = f"{absoluteBinaryPath} {' '.join(arguments)}" return self.run(final_command) diff --git a/src/processcdb/clangtidy.py b/src/processcdb/clangtidy.py index e766d37..82c76d7 100644 --- a/src/processcdb/clangtidy.py +++ b/src/processcdb/clangtidy.py @@ -11,9 +11,9 @@ from functools import partial import shutil import signal -from .logger import LOGGER as log -from .toolbase import Tool -from .tidy_converter import OutputParser +from logger import LOGGER as log +from toolbase import Tool +from tidy_converter import OutputParser list_of_futures = [] @@ -108,9 +108,8 @@ def _generate_cmd_queue(self, cdb, args): for compilation_unit in cdb: arguments = [] - arguments.extend(self.config.getlist("default_args")) directory = Path(compilation_unit["directory"]).absolute() - full_command = compilation_unit["command"].split(" ") + full_command = compilation_unit["arguments"] absolute_filename = directory / compilation_unit["file"] compiler = Path(full_command[0]).name.lower() @@ -120,14 +119,16 @@ def _generate_cmd_queue(self, cdb, args): arguments = self.convert_includes(arguments) arguments.extend(self.includes_as_cli_flags(self.default_includes())) - extra = "--quiet" - if compiler == "cl.exe": + default_args = self.config.getlist("default_args") + extra = f"--quiet {' '.join(default_args)}" + if compiler.endswith("cl.exe"): arguments = list(map(self.convert_arguments, arguments)) extra = f"{extra} --extra-arg-before=--driver-mode=cl" if absolute_filename.is_file(): if self.should_scan(absolute_filename, args.file): - tmp_cmd = f"cd {directory} && {self.binary} {extra} {absolute_filename} -- {' '.join(arguments)}" + absoluteBinaryPath = Path(self.binary).resolve() + tmp_cmd = f"cd {directory} && {absoluteBinaryPath} {extra} {absolute_filename} -- {' '.join(arguments)}" command_queue.append(tmp_cmd) else: log.debug(f"File {absolute_filename} is not scanned") diff --git a/src/processcdb/cppcheck.py b/src/processcdb/cppcheck.py index 2f3264d..53ccebc 100644 --- a/src/processcdb/cppcheck.py +++ b/src/processcdb/cppcheck.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from .toolbase import Tool +from toolbase import Tool import os import tempfile import configparser @@ -80,7 +80,8 @@ def execute(self, cdb, args): arguments.extend(["--xml", "--xml-version=2"]) if os.path.isfile(temp_name_includes) and os.path.isfile(temp_name_sources): - tmp_cmd = f"{self.binary} {' '.join(arguments)}" + absoluteBinaryPath = Path(self.binary).resolve() + tmp_cmd = f"{absoluteBinaryPath} {' '.join(arguments)}" if args.output is not None: final_command = f"{tmp_cmd} 2> {args.output}" else: diff --git a/src/processcdb/lizard.py b/src/processcdb/lizard.py index d6d894b..39f1bb6 100644 --- a/src/processcdb/lizard.py +++ b/src/processcdb/lizard.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from .toolbase import Tool +from toolbase import Tool import tempfile import os diff --git a/src/processcdb/misc.py b/src/processcdb/misc.py index 3418a2f..400d1e7 100644 --- a/src/processcdb/misc.py +++ b/src/processcdb/misc.py @@ -3,18 +3,10 @@ import platform import subprocess from pathlib import Path -from whatthepatch import parse_patch -from git import Repo -from appdirs import AppDirs from collections import ChainMap import argparse -from ._version import get_versions -from .logger import LOGGER as log, LOG_LEVELS # noqa: F401 - -__author__ = "Jani Mikkonen" -__email__ = "jani.mikkonen@gmail.com" -__version__ = get_versions()["version"] - +from _version import get_versions +from logger import LOGGER as log, LOG_LEVELS # noqa: F401 def is_windows(): return "WINDOWS" in platform.system().upper() @@ -49,37 +41,6 @@ def capture_output(args, captureErr=True, capture_exceptions=True): return buff.split("\n") -def remove_untouched_files(cdb, commits): - def modified_lines(data): - return data[0] is None and data[1] is not None - - def line_numbers(data): - return data[1] - - patch = None - repo = Repo(".", search_parent_directories=True) - commits = list(filter(None, commits)) - if len(commits) == 2: - patch = parse_patch(repo.git.diff(commits[0], commits[1])) - else: - patch = parse_patch(repo.git.diff(f"{commits[0]}^")) - new_cdb = [] - base_dir = Path(repo.git_dir).parent - lookup = {} - for i in cdb: - current = Path(i["directory"]) / i["file"] - lookup[current] = i - - for diff in patch: - fullname = base_dir / diff.header.new_path - fullname = (base_dir / diff.header.new_path).absolute() - if fullname in lookup: - cdb_entry = lookup[fullname] - cdb_entry["changes_rows"] = map(line_numbers, filter(modified_lines, diff.changes)) - new_cdb.append(cdb_entry) - return new_cdb - - def remove_dupes(cdb): seen = [] new_cdb = [] @@ -93,8 +54,7 @@ def remove_dupes(cdb): def argument_parser(tools): # TODO: Offload tool specific argument to the tool class itself if possible. - app_dirs = AppDirs("processcdb", __author__) - default_config_file = Path(app_dirs.user_config_dir) / "processcdb.ini" + default_config_file = "processcdb.ini" parser = argparse.ArgumentParser(description="Static analysis wrapper", epilog=f"Available tools: \n{','.join(tools.keys())}") parser.add_argument( "--cdb", @@ -204,6 +164,9 @@ def to_list(value): def to_dict(value): + if value == None or value == '': + return None + def format(val): k,v = val.split("=", 2) return {k: v.split(",")} diff --git a/src/processcdb/toolbase.py b/src/processcdb/toolbase.py index 7c88494..545d48c 100644 --- a/src/processcdb/toolbase.py +++ b/src/processcdb/toolbase.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -from .misc import is_any_windows, capture_output, to_dict, to_list +from misc import is_any_windows, capture_output, to_dict, to_list import multiprocessing import subprocess import configparser import os -from .logger import LOGGER as log +from logger import LOGGER as log import shutil import shlex from fnmatch import fnmatch @@ -129,24 +129,36 @@ def run(self, command_line): return subprocess.call(command_line, shell=True) def include_string(self, include_path, path_matchers): - if any([fnmatch(include_path, pattern) for pattern in path_matchers]): + if len(path_matchers[0]) > 0 and any([fnmatch(include_path, pattern) for pattern in path_matchers]): return f'-isystem "{include_path}"' else: - return f"-I{include_path}" + return f'-I"{include_path}"' def includes_as_cli_flags(self, includes): path_matchers = self.config.getlist("includes_as_system") return list(map(lambda i: self.include_string(i, path_matchers), includes)) def convert_includes(self, arguments): - def convert(arg, path_matchers): - if arg and arg.startswith("-I"): - return self.include_string(arg[2:], path_matchers) - return arg - + def convert(arg, foundInclude, path_matchers): + if foundInclude: + foundInclude = False + return self.include_string(arg, path_matchers), foundInclude + elif arg and arg.startswith("-I"): + foundInclude = True + return None, foundInclude + return arg, foundInclude + + foundInclude = False path_matchers = self.config.getlist("includes_as_system") - return list(map(lambda arg: convert(arg, path_matchers), arguments)) - + + result = [] + for arg in arguments: + converted = convert(arg, foundInclude, path_matchers) + if converted[0] != None: + result.append(converted[0]) + foundInclude = converted[1] + + return result def filter_arguments(self, args): def allowed_argument(arg): @@ -157,8 +169,9 @@ def allowed_argument(arg): def add_additions(self, args): new_args = args.copy() additions = self.config.getdict("arg_additions") - for arg in args: - key = arg[1:] - if key in additions: - new_args.extend(additions[key]) + if additions != None: + for arg in args: + key = arg[1:] + if key in additions: + new_args.extend(additions[key]) return new_args