From 544ff1f64c0fc8ec92b7531705838b1e90339cb1 Mon Sep 17 00:00:00 2001 From: David Crook Date: Fri, 1 Jan 2021 14:32:21 +0000 Subject: [PATCH 01/13] Remove all . from import paths to solve "attempted relative import with no known parent package" --- src/processcdb/__init__.py | 2 +- src/processcdb/__main__.py | 6 +++--- src/processcdb/cdb_tools.py | 10 +++++----- src/processcdb/clang.py | 4 ++-- src/processcdb/clangtidy.py | 6 +++--- src/processcdb/cppcheck.py | 2 +- src/processcdb/lizard.py | 2 +- src/processcdb/misc.py | 4 ++-- src/processcdb/toolbase.py | 4 ++-- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/processcdb/__init__.py b/src/processcdb/__init__.py index facf1ee..9db673b 100644 --- a/src/processcdb/__init__.py +++ b/src/processcdb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ._version import get_versions +from _version import get_versions __version__ = get_versions()["version"] del get_versions diff --git a/src/processcdb/__main__.py b/src/processcdb/__main__.py index ce782ed..467b410 100755 --- a/src/processcdb/__main__.py +++ b/src/processcdb/__main__.py @@ -3,10 +3,10 @@ import sys import json import traceback -from .cdb_tools import TOOLS -from .misc import remove_untouched_files, remove_dupes, argument_parser +from cdb_tools import TOOLS +from misc import remove_untouched_files, remove_dupes, argument_parser from configparser import ConfigParser -from .logger import LOGGER as log, LOG_LEVELS +from logger import LOGGER as log, LOG_LEVELS def main(): diff --git a/src/processcdb/cdb_tools.py b/src/processcdb/cdb_tools.py index 8387cf4..5813391 100644 --- a/src/processcdb/cdb_tools.py +++ b/src/processcdb/cdb_tools.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -from .toolbase import Tool # noqa: F401 -from .clangtidy import ClangTidy # noqa: F401 -from .clang import Clang # noqa: F401 -from .lizard import Lizard # noqa: F401 -from .cppcheck import CppCheck # noqa: F401 +from toolbase import Tool # noqa: F401 +from clangtidy import ClangTidy # noqa: F401 +from clang import Clang # noqa: F401 +from lizard import Lizard # noqa: F401 +from cppcheck import CppCheck # noqa: F401 TOOLS = { "clang-tidy": ClangTidy, diff --git a/src/processcdb/clang.py b/src/processcdb/clang.py index 78266d1..4f70b01 100644 --- a/src/processcdb/clang.py +++ b/src/processcdb/clang.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from .misc import is_any_windows -from .toolbase import Tool +from misc import is_any_windows +from toolbase import Tool class Clang(Tool): diff --git a/src/processcdb/clangtidy.py b/src/processcdb/clangtidy.py index e766d37..2129573 100644 --- a/src/processcdb/clangtidy.py +++ b/src/processcdb/clangtidy.py @@ -11,9 +11,9 @@ from functools import partial import shutil import signal -from .logger import LOGGER as log -from .toolbase import Tool -from .tidy_converter import OutputParser +from logger import LOGGER as log +from toolbase import Tool +from tidy_converter import OutputParser list_of_futures = [] diff --git a/src/processcdb/cppcheck.py b/src/processcdb/cppcheck.py index 2f3264d..d7b46ed 100644 --- a/src/processcdb/cppcheck.py +++ b/src/processcdb/cppcheck.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from .toolbase import Tool +from toolbase import Tool import os import tempfile import configparser diff --git a/src/processcdb/lizard.py b/src/processcdb/lizard.py index d6d894b..39f1bb6 100644 --- a/src/processcdb/lizard.py +++ b/src/processcdb/lizard.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from .toolbase import Tool +from toolbase import Tool import tempfile import os diff --git a/src/processcdb/misc.py b/src/processcdb/misc.py index 3418a2f..6a139b6 100644 --- a/src/processcdb/misc.py +++ b/src/processcdb/misc.py @@ -8,8 +8,8 @@ from appdirs import AppDirs from collections import ChainMap import argparse -from ._version import get_versions -from .logger import LOGGER as log, LOG_LEVELS # noqa: F401 +from _version import get_versions +from logger import LOGGER as log, LOG_LEVELS # noqa: F401 __author__ = "Jani Mikkonen" __email__ = "jani.mikkonen@gmail.com" diff --git a/src/processcdb/toolbase.py b/src/processcdb/toolbase.py index 7c88494..73ad48e 100644 --- a/src/processcdb/toolbase.py +++ b/src/processcdb/toolbase.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -from .misc import is_any_windows, capture_output, to_dict, to_list +from misc import is_any_windows, capture_output, to_dict, to_list import multiprocessing import subprocess import configparser import os -from .logger import LOGGER as log +from logger import LOGGER as log import shutil import shlex from fnmatch import fnmatch From 03e7d60c2b6ba44be5d2c6a3561d3851d5c156c4 Mon Sep 17 00:00:00 2001 From: David Crook Date: Fri, 1 Jan 2021 16:43:23 +0000 Subject: [PATCH 02/13] Removed user config folder When dumping the config this is now written to the current working directory instead of Janis specific folder --- src/processcdb/misc.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/processcdb/misc.py b/src/processcdb/misc.py index 6a139b6..f0efccb 100644 --- a/src/processcdb/misc.py +++ b/src/processcdb/misc.py @@ -11,11 +11,6 @@ from _version import get_versions from logger import LOGGER as log, LOG_LEVELS # noqa: F401 -__author__ = "Jani Mikkonen" -__email__ = "jani.mikkonen@gmail.com" -__version__ = get_versions()["version"] - - def is_windows(): return "WINDOWS" in platform.system().upper() @@ -93,8 +88,7 @@ def remove_dupes(cdb): def argument_parser(tools): # TODO: Offload tool specific argument to the tool class itself if possible. - app_dirs = AppDirs("processcdb", __author__) - default_config_file = Path(app_dirs.user_config_dir) / "processcdb.ini" + default_config_file = "processcdb.ini" parser = argparse.ArgumentParser(description="Static analysis wrapper", epilog=f"Available tools: \n{','.join(tools.keys())}") parser.add_argument( "--cdb", From 38f73e49f78e9a071a3919a6ff92ffedf7e6ba53 Mon Sep 17 00:00:00 2001 From: David Crook Date: Fri, 1 Jan 2021 16:44:05 +0000 Subject: [PATCH 03/13] Ignore *.ini Updated the ignore file to make sure the processcdb.ini is not added into git --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c0fb662..02a6304 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ venv/ dist/ *.egg-info/ +*.ini \ No newline at end of file From f2403d5930863b2f56e2e2919fb1919b6cc5c15c Mon Sep 17 00:00:00 2001 From: David Crook Date: Fri, 1 Jan 2021 18:10:17 +0000 Subject: [PATCH 04/13] Update to make clang tidy run A couple of issues were found when trying to make clang tidy run. * The compiliation database was trying to use "command" instead of "arguments". According to the docs either are possible to this probably should be updated to support both instead of the quick fix I did here * to_dict can get given an empty string. Handle that error and return correctly * If there are no arg_additions specificed in the config then this needs to handle the error gracefully --- src/processcdb/clangtidy.py | 2 +- src/processcdb/misc.py | 3 +++ src/processcdb/toolbase.py | 9 +++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/processcdb/clangtidy.py b/src/processcdb/clangtidy.py index 2129573..1b4da9a 100644 --- a/src/processcdb/clangtidy.py +++ b/src/processcdb/clangtidy.py @@ -110,7 +110,7 @@ def _generate_cmd_queue(self, cdb, args): arguments.extend(self.config.getlist("default_args")) directory = Path(compilation_unit["directory"]).absolute() - full_command = compilation_unit["command"].split(" ") + full_command = compilation_unit["arguments"] absolute_filename = directory / compilation_unit["file"] compiler = Path(full_command[0]).name.lower() diff --git a/src/processcdb/misc.py b/src/processcdb/misc.py index f0efccb..ca352b1 100644 --- a/src/processcdb/misc.py +++ b/src/processcdb/misc.py @@ -198,6 +198,9 @@ def to_list(value): def to_dict(value): + if value == None or value == '': + return None + def format(val): k,v = val.split("=", 2) return {k: v.split(",")} diff --git a/src/processcdb/toolbase.py b/src/processcdb/toolbase.py index 73ad48e..81f17e5 100644 --- a/src/processcdb/toolbase.py +++ b/src/processcdb/toolbase.py @@ -157,8 +157,9 @@ def allowed_argument(arg): def add_additions(self, args): new_args = args.copy() additions = self.config.getdict("arg_additions") - for arg in args: - key = arg[1:] - if key in additions: - new_args.extend(additions[key]) + if additions != None: + for arg in args: + key = arg[1:] + if key in additions: + new_args.extend(additions[key]) return new_args From c398bd87a123add974d0de292fe2c934a999d5ef Mon Sep 17 00:00:00 2001 From: David Crook Date: Fri, 1 Jan 2021 23:03:58 +0000 Subject: [PATCH 05/13] Ignore *.xml files xml files are the output from the process and so should not be submitted. This name is custom though so this may not be totally correct in all cases --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 02a6304..6ad8696 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ __pycache__/ venv/ dist/ *.egg-info/ -*.ini \ No newline at end of file +*.ini +*.xml \ No newline at end of file From 971f0e8e37c7b74a81d67a901a50905a61899d0e Mon Sep 17 00:00:00 2001 From: David Crook Date: Fri, 1 Jan 2021 23:45:42 +0000 Subject: [PATCH 06/13] Update convert_includes * If no path_matcher is given then it would default to everything using the path matcher. Now it defaults to nothing using it instead as the setting is additive * It used to try to find the include path by using arg[2:] but will only work when the arguments are a single string. As the -I and the include_path are seperate arguments this needed to be rewritten to combine these into one argument. Sadly lambdas in python cannot handle out parameters or multiple return values so this needed to be unrolled a bit --- src/processcdb/toolbase.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/processcdb/toolbase.py b/src/processcdb/toolbase.py index 81f17e5..545d48c 100644 --- a/src/processcdb/toolbase.py +++ b/src/processcdb/toolbase.py @@ -129,24 +129,36 @@ def run(self, command_line): return subprocess.call(command_line, shell=True) def include_string(self, include_path, path_matchers): - if any([fnmatch(include_path, pattern) for pattern in path_matchers]): + if len(path_matchers[0]) > 0 and any([fnmatch(include_path, pattern) for pattern in path_matchers]): return f'-isystem "{include_path}"' else: - return f"-I{include_path}" + return f'-I"{include_path}"' def includes_as_cli_flags(self, includes): path_matchers = self.config.getlist("includes_as_system") return list(map(lambda i: self.include_string(i, path_matchers), includes)) def convert_includes(self, arguments): - def convert(arg, path_matchers): - if arg and arg.startswith("-I"): - return self.include_string(arg[2:], path_matchers) - return arg - + def convert(arg, foundInclude, path_matchers): + if foundInclude: + foundInclude = False + return self.include_string(arg, path_matchers), foundInclude + elif arg and arg.startswith("-I"): + foundInclude = True + return None, foundInclude + return arg, foundInclude + + foundInclude = False path_matchers = self.config.getlist("includes_as_system") - return list(map(lambda arg: convert(arg, path_matchers), arguments)) - + + result = [] + for arg in arguments: + converted = convert(arg, foundInclude, path_matchers) + if converted[0] != None: + result.append(converted[0]) + foundInclude = converted[1] + + return result def filter_arguments(self, args): def allowed_argument(arg): From b6d43aead808129514c1939e6c2aa7e5ff14a800 Mon Sep 17 00:00:00 2001 From: David Crook Date: Fri, 1 Jan 2021 23:57:55 +0000 Subject: [PATCH 07/13] Update README.md Fixed a few spelling mistakes in the readme and added a note that the tools will all be defined in the config file --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b2b1127..362b376 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ After a process of generating a compile_commands.json, you can run processcdb wi processcdb --tool clang-tidy -This will try to locate the json file from current working directory and runs the tool, in this case -clang-tidy, against all files that are compiled and not blacklisted in processcdb comfig file or in +This will try to locate the json file from current working directory and runs the tool, in this case the +clang-tidy tool as defined in the config file, against all files that are compiled and not blacklisted in processcdb config file or in tools own configuration file and generates the output to standard output. If you need to run the tool when you don't have access to change the current working directory, you can pass `--cdb` and absolute location: @@ -77,7 +77,7 @@ capture the standard output or provide `--config` parameter. ## Configuration file Each tool has a separate section and each section can be configured either in the tool specific section or -in default. Minimal. single tool configuratio would look something like this: +in default. The minimal single tool configuration would look something like this: [clang-tidy] binary=C:\llvm-11.0.0\bin\clang-tidy.exe From b7299e6888727ef853c40b218ab7d9cd143855e5 Mon Sep 17 00:00:00 2001 From: David Crook Date: Sat, 2 Jan 2021 15:47:07 +0000 Subject: [PATCH 08/13] Deleted git changelist support This was used to limit the compile_commands to specific git changelists. While something like this may be needed in the future this will need to be rewritten for perforce. The --commit arguments still exist but I have commented out the implementation for now until we know what is going to be needed for this fully. The main reason to remove this is to remove the dependencies and allow us to clean up our python packages --- src/processcdb/__main__.py | 5 ++--- src/processcdb/misc.py | 34 ---------------------------------- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/src/processcdb/__main__.py b/src/processcdb/__main__.py index 467b410..96ce6b8 100755 --- a/src/processcdb/__main__.py +++ b/src/processcdb/__main__.py @@ -4,11 +4,10 @@ import json import traceback from cdb_tools import TOOLS -from misc import remove_untouched_files, remove_dupes, argument_parser +from misc import remove_dupes, argument_parser from configparser import ConfigParser from logger import LOGGER as log, LOG_LEVELS - def main(): args = argument_parser(TOOLS).parse_args() log.setLevel(LOG_LEVELS[args.loglevel]) @@ -40,7 +39,7 @@ def main(): cdb = json.loads(args.cdb.read_text()) if cdb: if args.commit_a is not None: - cdb = remove_untouched_files(cdb, (args.commit_a, args.commit_b)) + #cdb = filterByChangelist(cdb, (args.commit_a, args.commit_b)) if not args.allow_dupes: cdb = remove_dupes(cdb) diff --git a/src/processcdb/misc.py b/src/processcdb/misc.py index ca352b1..400d1e7 100644 --- a/src/processcdb/misc.py +++ b/src/processcdb/misc.py @@ -3,9 +3,6 @@ import platform import subprocess from pathlib import Path -from whatthepatch import parse_patch -from git import Repo -from appdirs import AppDirs from collections import ChainMap import argparse from _version import get_versions @@ -44,37 +41,6 @@ def capture_output(args, captureErr=True, capture_exceptions=True): return buff.split("\n") -def remove_untouched_files(cdb, commits): - def modified_lines(data): - return data[0] is None and data[1] is not None - - def line_numbers(data): - return data[1] - - patch = None - repo = Repo(".", search_parent_directories=True) - commits = list(filter(None, commits)) - if len(commits) == 2: - patch = parse_patch(repo.git.diff(commits[0], commits[1])) - else: - patch = parse_patch(repo.git.diff(f"{commits[0]}^")) - new_cdb = [] - base_dir = Path(repo.git_dir).parent - lookup = {} - for i in cdb: - current = Path(i["directory"]) / i["file"] - lookup[current] = i - - for diff in patch: - fullname = base_dir / diff.header.new_path - fullname = (base_dir / diff.header.new_path).absolute() - if fullname in lookup: - cdb_entry = lookup[fullname] - cdb_entry["changes_rows"] = map(line_numbers, filter(modified_lines, diff.changes)) - new_cdb.append(cdb_entry) - return new_cdb - - def remove_dupes(cdb): seen = [] new_cdb = [] From 26860f910ef6e11ec40c2d3ae637b0182d561d14 Mon Sep 17 00:00:00 2001 From: David Crook Date: Sat, 2 Jan 2021 15:48:08 +0000 Subject: [PATCH 09/13] Commented out check too Fixed compile error --- src/processcdb/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processcdb/__main__.py b/src/processcdb/__main__.py index 96ce6b8..b5bce86 100755 --- a/src/processcdb/__main__.py +++ b/src/processcdb/__main__.py @@ -38,7 +38,7 @@ def main(): if args.cdb.is_file(): cdb = json.loads(args.cdb.read_text()) if cdb: - if args.commit_a is not None: + #if args.commit_a is not None: #cdb = filterByChangelist(cdb, (args.commit_a, args.commit_b)) if not args.allow_dupes: From 7cf212e332f33cfc3fbb76d16c1cc98836261b96 Mon Sep 17 00:00:00 2001 From: David Crook Date: Sat, 2 Jan 2021 15:49:11 +0000 Subject: [PATCH 10/13] Add run bat Add an initial batch script to run this tool for or usages. The paths are currently hard coded but when we move this into the full depot then we can abstract these correctly --- run_processcdb.bat | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 run_processcdb.bat diff --git a/run_processcdb.bat b/run_processcdb.bat new file mode 100644 index 0000000..3981a45 --- /dev/null +++ b/run_processcdb.bat @@ -0,0 +1,6 @@ +@echo off +pushd + +D:\fone\branch\external\ego\dev\python3\bin\python.exe src/processcdb --tool clang-tidy --xml --output results.xml --cdb D:\fone\branch\external\ego\dev\clang_tools\compile_commands.json + +popd \ No newline at end of file From 28e4b26591dcff05e37a9f7d34a67dacf3fa9142 Mon Sep 17 00:00:00 2001 From: David Crook Date: Sun, 3 Jan 2021 14:48:37 +0000 Subject: [PATCH 11/13] Make tool binary paths absolute If you put a path ../clang-tidy.exe into the binary field then it was not getting found and now it will correctly. Only clang-tody has been tested though --- src/processcdb/clang.py | 6 ++++-- src/processcdb/clangtidy.py | 3 ++- src/processcdb/cppcheck.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/processcdb/clang.py b/src/processcdb/clang.py index 4f70b01..7c041fd 100644 --- a/src/processcdb/clang.py +++ b/src/processcdb/clang.py @@ -16,8 +16,10 @@ def execute(self, cdb, args): if args.output is not None: arguments.extend([f"--output {args.output}"]) + absoluteBinaryPath = Path(self.binary).resolve() + if is_any_windows(): - arguments.extend([f"--use-analyzer {self.binary}"]) + arguments.extend([f"--use-analyzer {absoluteBinaryPath}"]) - final_command = f"{self.binary} {' '.join(arguments)}" + final_command = f"{absoluteBinaryPath} {' '.join(arguments)}" return self.run(final_command) diff --git a/src/processcdb/clangtidy.py b/src/processcdb/clangtidy.py index 1b4da9a..07d3e97 100644 --- a/src/processcdb/clangtidy.py +++ b/src/processcdb/clangtidy.py @@ -127,7 +127,8 @@ def _generate_cmd_queue(self, cdb, args): if absolute_filename.is_file(): if self.should_scan(absolute_filename, args.file): - tmp_cmd = f"cd {directory} && {self.binary} {extra} {absolute_filename} -- {' '.join(arguments)}" + absoluteBinaryPath = Path(self.binary).resolve() + tmp_cmd = f"cd {directory} && {absoluteBinaryPath} {extra} {absolute_filename} -- {' '.join(arguments)}" command_queue.append(tmp_cmd) else: log.debug(f"File {absolute_filename} is not scanned") diff --git a/src/processcdb/cppcheck.py b/src/processcdb/cppcheck.py index d7b46ed..53ccebc 100644 --- a/src/processcdb/cppcheck.py +++ b/src/processcdb/cppcheck.py @@ -80,7 +80,8 @@ def execute(self, cdb, args): arguments.extend(["--xml", "--xml-version=2"]) if os.path.isfile(temp_name_includes) and os.path.isfile(temp_name_sources): - tmp_cmd = f"{self.binary} {' '.join(arguments)}" + absoluteBinaryPath = Path(self.binary).resolve() + tmp_cmd = f"{absoluteBinaryPath} {' '.join(arguments)}" if args.output is not None: final_command = f"{tmp_cmd} 2> {args.output}" else: From 763d2aefb2d9105a70cdb6ceddb4259ff3060912 Mon Sep 17 00:00:00 2001 From: David Crook Date: Mon, 4 Jan 2021 12:23:40 +0000 Subject: [PATCH 12/13] Fixed default_args in clang-tidy default_args in clang-tidy were getting prepended onto the standard set of arguments but this meant that filter_arguments culls the first one as it has an args[1:] in it. Secondly these were getting put after the -- line and so now these are "extra" instead and passed directly to clang-tidy instead of as extra arguments that are passed to the compiler --- src/processcdb/clangtidy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/processcdb/clangtidy.py b/src/processcdb/clangtidy.py index 07d3e97..6494757 100644 --- a/src/processcdb/clangtidy.py +++ b/src/processcdb/clangtidy.py @@ -108,7 +108,6 @@ def _generate_cmd_queue(self, cdb, args): for compilation_unit in cdb: arguments = [] - arguments.extend(self.config.getlist("default_args")) directory = Path(compilation_unit["directory"]).absolute() full_command = compilation_unit["arguments"] absolute_filename = directory / compilation_unit["file"] @@ -120,7 +119,8 @@ def _generate_cmd_queue(self, cdb, args): arguments = self.convert_includes(arguments) arguments.extend(self.includes_as_cli_flags(self.default_includes())) - extra = "--quiet" + default_args = self.config.getlist("default_args") + extra = f"--quiet {' '.join(default_args)}" if compiler == "cl.exe": arguments = list(map(self.convert_arguments, arguments)) extra = f"{extra} --extra-arg-before=--driver-mode=cl" From d047acc4b8b5a1627c3df345cb6c9dfdb164b501 Mon Sep 17 00:00:00 2001 From: David Crook Date: Tue, 5 Jan 2021 11:07:36 +0000 Subject: [PATCH 13/13] Made clang-cl.exe set the driver mode correctly Made the clang tidy processor work with cl.exe and clang-cl.exe as this is what is available on windows --- src/processcdb/clangtidy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processcdb/clangtidy.py b/src/processcdb/clangtidy.py index 6494757..82c76d7 100644 --- a/src/processcdb/clangtidy.py +++ b/src/processcdb/clangtidy.py @@ -121,7 +121,7 @@ def _generate_cmd_queue(self, cdb, args): default_args = self.config.getlist("default_args") extra = f"--quiet {' '.join(default_args)}" - if compiler == "cl.exe": + if compiler.endswith("cl.exe"): arguments = list(map(self.convert_arguments, arguments)) extra = f"{extra} --extra-arg-before=--driver-mode=cl"