From 1be9fc41982bd361aec0ac819aae2711d30d8ab0 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 24 Jun 2025 13:16:09 +0200 Subject: [PATCH 01/15] Resolve symlinks in plist files not created by clang-tidy --- src/codechecker_script.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index 43f6227d..df9b664b 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -280,13 +280,12 @@ def resolve_symlinks(): files_processed = 0 for root, _, files in os.walk(analyze_outdir): for filename in files: - if re.search("clang-tidy", filename): - filepath = os.path.join(root, filename) - if os.path.splitext(filepath)[1] == ".plist": - resolve_plist_symlinks(filepath) - elif os.path.splitext(filepath)[1] == ".yaml": - resolve_yaml_symlinks(filepath) - files_processed += 1 + filepath = os.path.join(root, filename) + if os.path.splitext(filepath)[1] == ".plist": + resolve_plist_symlinks(filepath) + elif os.path.splitext(filepath)[1] == ".yaml": + resolve_yaml_symlinks(filepath) + files_processed += 1 logging.info("Processed file paths in %d files", files_processed) def update_file_paths(): """ Fix bazel sandbox paths and resolve symbolic links in generated files to real paths """ From 540546ab6eeb54650dcd32b48b59fccbb06f538c Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 25 Aug 2025 07:35:16 +0200 Subject: [PATCH 02/15] Activate test for codechecker --- test/unit/virtual_include/test_virtual_include.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/unit/virtual_include/test_virtual_include.py b/test/unit/virtual_include/test_virtual_include.py index 524d361c..145acb4a 100644 --- a/test/unit/virtual_include/test_virtual_include.py +++ b/test/unit/virtual_include/test_virtual_include.py @@ -102,9 +102,7 @@ def test_bazel_codechecker_plist_path_resolved(self): self.assertTrue( os.path.isdir(f"{self.BAZEL_BIN_DIR}/_virtual_includes") ) - # FIXME: In the postprocessed plists, all _virtual_include paths - # should've been removed. Possible fix is in the github PR #14. - self.assertNotEqual( + self.assertEqual( self.contains_in_files(r"/_virtual_includes/", plist_files), [] ) From 9dd3fdadebc7ca8bb2c3f3abe121c43df9d49d8d Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 25 Aug 2025 12:43:54 +0200 Subject: [PATCH 03/15] Add prefix to be stripped --- src/codechecker_script.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index df9b664b..b223c5e9 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -32,6 +32,7 @@ r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", + r"bazel_codechecker\/": "", } From 25cca5fb4cc577b94629e675c8b78987d64b41df Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 25 Aug 2025 12:48:23 +0200 Subject: [PATCH 04/15] Fix for ubuntu systems --- src/codechecker_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index b223c5e9..58ef64f4 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -32,7 +32,7 @@ r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", - r"bazel_codechecker\/": "", + r"^bazel_codechecker\/": "", } From 4b4bb49d60ad60508f0f43a914b5034b8d46d077 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 25 Aug 2025 12:53:10 +0200 Subject: [PATCH 05/15] Fix regex, its not the first in the line --- src/codechecker_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index 58ef64f4..c2336257 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -32,7 +32,7 @@ r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", - r"^bazel_codechecker\/": "", + r"bazel_codechecker\/": "", } From afc7296f1e436f88406aed2ba28a2ca53d1b0235 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 25 Aug 2025 14:11:14 +0200 Subject: [PATCH 06/15] Add explanation --- src/codechecker_script.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index c2336257..85dca88a 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -32,6 +32,11 @@ r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", + # The path getting into the plist file on a docker img is: + # bazel_codechecker/bazel-out/k8-fastbuild/... + # But on regular other system its: + # /home/.../execroot/bazel_codechecker/bazel-out/k8-fastbuild/... + # This is because normally the runner is not the root user r"bazel_codechecker\/": "", } From 90ecb5c2b9e66b917df07128a232c17e2f86aa31 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 25 Aug 2025 15:00:48 +0200 Subject: [PATCH 07/15] Correct comment --- src/codechecker_script.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index 85dca88a..4597f7ad 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -32,11 +32,10 @@ r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", - # The path getting into the plist file on a docker img is: - # bazel_codechecker/bazel-out/k8-fastbuild/... - # But on regular other system its: - # /home/.../execroot/bazel_codechecker/bazel-out/k8-fastbuild/... - # This is because normally the runner is not the root user + # Running bazel with --spawn_strategy=processwrapper-sandbox + # results in its path being wrongly replaced! + # In that case the 1st then 3rd regex will match and leave the path in a + # half baked state, this regex just finishes the job r"bazel_codechecker\/": "", } From a0a536f26e36267d9c0ab388a78a79e9d4918d0e Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 26 Aug 2025 08:54:12 +0200 Subject: [PATCH 08/15] Remove unnecesary step causing errors --- src/codechecker_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index 4597f7ad..b9554c83 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -294,7 +294,7 @@ def resolve_symlinks(): logging.info("Processed file paths in %d files", files_processed) def update_file_paths(): """ Fix bazel sandbox paths and resolve symbolic links in generated files to real paths """ - fix_bazel_paths() + #fix_bazel_paths() # this is unnecessary resolve_symlinks() From 6af6175287628a78fd6f6e4a33bc44c630e58a37 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 26 Aug 2025 09:06:02 +0200 Subject: [PATCH 09/15] Add comments --- src/codechecker_script.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index b9554c83..b4343bf8 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -27,6 +27,7 @@ CODECHECKER_ENV = "{codechecker_env}" COMPILE_COMMANDS = "{compile_commands}" +# Note: unused START_PATH = r"\/(?:(?!\.\s+)\S)+" BAZEL_PATHS = { r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", @@ -200,6 +201,7 @@ def analyze(): def fix_bazel_paths(): """ Remove Bazel leading paths in all files """ + # Note: unused stage("Fix CodeChecker output:") folder = CODECHECKER_FILES logging.info("Fixing Bazel paths in %s", folder) From 26d382dc34187c9cddf81491145dfe9ad2dfb19e Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 26 Aug 2025 14:34:22 +0200 Subject: [PATCH 10/15] remove regex modifications --- src/codechecker_script.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index b4343bf8..7837fe45 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -27,19 +27,6 @@ CODECHECKER_ENV = "{codechecker_env}" COMPILE_COMMANDS = "{compile_commands}" -# Note: unused -START_PATH = r"\/(?:(?!\.\s+)\S)+" -BAZEL_PATHS = { - r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", - START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", - START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", - # Running bazel with --spawn_strategy=processwrapper-sandbox - # results in its path being wrongly replaced! - # In that case the 1st then 3rd regex will match and leave the path in a - # half baked state, this regex just finishes the job - r"bazel_codechecker\/": "", -} - def fail(message, exit_code=1): """ Print error message and return exit code """ @@ -199,26 +186,6 @@ def analyze(): fail("Make sure that the target can be built first") -def fix_bazel_paths(): - """ Remove Bazel leading paths in all files """ - # Note: unused - stage("Fix CodeChecker output:") - folder = CODECHECKER_FILES - logging.info("Fixing Bazel paths in %s", folder) - counter = 0 - for root, _, files in os.walk(folder): - for filename in files: - fullpath = os.path.join(root, filename) - with open(fullpath, "rt") as data_file: - data = data_file.read() - for pattern, replace in BAZEL_PATHS.items(): - data = re.sub(pattern, replace, data) - with open(fullpath, "w") as data_file: - data_file.write(data) - counter += 1 - logging.info("Fixed Bazel paths in %d files", counter) - - def realpath(filename): """ Return real full absolute path for given filename """ if os.path.exists(filename): @@ -296,7 +263,6 @@ def resolve_symlinks(): logging.info("Processed file paths in %d files", files_processed) def update_file_paths(): """ Fix bazel sandbox paths and resolve symbolic links in generated files to real paths """ - #fix_bazel_paths() # this is unnecessary resolve_symlinks() From 36a802f5b1d17886caab484cdc522e4fd6836e8f Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Thu, 28 Aug 2025 16:02:40 +0200 Subject: [PATCH 11/15] Revert "remove regex modifications" This reverts commit c5f47bf98b32a7723788486c266676500452b191. --- src/codechecker_script.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index 7837fe45..b4343bf8 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -27,6 +27,19 @@ CODECHECKER_ENV = "{codechecker_env}" COMPILE_COMMANDS = "{compile_commands}" +# Note: unused +START_PATH = r"\/(?:(?!\.\s+)\S)+" +BAZEL_PATHS = { + r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", + START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", + START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", + # Running bazel with --spawn_strategy=processwrapper-sandbox + # results in its path being wrongly replaced! + # In that case the 1st then 3rd regex will match and leave the path in a + # half baked state, this regex just finishes the job + r"bazel_codechecker\/": "", +} + def fail(message, exit_code=1): """ Print error message and return exit code """ @@ -186,6 +199,26 @@ def analyze(): fail("Make sure that the target can be built first") +def fix_bazel_paths(): + """ Remove Bazel leading paths in all files """ + # Note: unused + stage("Fix CodeChecker output:") + folder = CODECHECKER_FILES + logging.info("Fixing Bazel paths in %s", folder) + counter = 0 + for root, _, files in os.walk(folder): + for filename in files: + fullpath = os.path.join(root, filename) + with open(fullpath, "rt") as data_file: + data = data_file.read() + for pattern, replace in BAZEL_PATHS.items(): + data = re.sub(pattern, replace, data) + with open(fullpath, "w") as data_file: + data_file.write(data) + counter += 1 + logging.info("Fixed Bazel paths in %d files", counter) + + def realpath(filename): """ Return real full absolute path for given filename """ if os.path.exists(filename): @@ -263,6 +296,7 @@ def resolve_symlinks(): logging.info("Processed file paths in %d files", files_processed) def update_file_paths(): """ Fix bazel sandbox paths and resolve symbolic links in generated files to real paths """ + #fix_bazel_paths() # this is unnecessary resolve_symlinks() From 980cefd4b4c6f807910d6d98e258d3e45143dfb1 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Thu, 28 Aug 2025 16:02:49 +0200 Subject: [PATCH 12/15] Revert "Add comments" This reverts commit e6e2c27fdfb7f20cf45d9e8adbde9127e3445894. --- src/codechecker_script.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index b4343bf8..b9554c83 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -27,7 +27,6 @@ CODECHECKER_ENV = "{codechecker_env}" COMPILE_COMMANDS = "{compile_commands}" -# Note: unused START_PATH = r"\/(?:(?!\.\s+)\S)+" BAZEL_PATHS = { r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", @@ -201,7 +200,6 @@ def analyze(): def fix_bazel_paths(): """ Remove Bazel leading paths in all files """ - # Note: unused stage("Fix CodeChecker output:") folder = CODECHECKER_FILES logging.info("Fixing Bazel paths in %s", folder) From e0da77ff0e43e67331a47c2cce6df06923b4038d Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Thu, 28 Aug 2025 16:02:55 +0200 Subject: [PATCH 13/15] Revert "Remove unnecesary step causing errors" This reverts commit d3a39e39be802fd0d12ee1a5644769122018b190. --- src/codechecker_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index b9554c83..4597f7ad 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -294,7 +294,7 @@ def resolve_symlinks(): logging.info("Processed file paths in %d files", files_processed) def update_file_paths(): """ Fix bazel sandbox paths and resolve symbolic links in generated files to real paths """ - #fix_bazel_paths() # this is unnecessary + fix_bazel_paths() resolve_symlinks() From 4691ba1ffa871f219e86b56cd03fff1f48065a0c Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Fri, 29 Aug 2025 09:05:53 +0200 Subject: [PATCH 14/15] Change order of steps --- src/codechecker_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index 4597f7ad..5ea868be 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -293,9 +293,9 @@ def resolve_symlinks(): files_processed += 1 logging.info("Processed file paths in %d files", files_processed) def update_file_paths(): - """ Fix bazel sandbox paths and resolve symbolic links in generated files to real paths """ - fix_bazel_paths() + """ Resolve symlinks from local jobs, then try fixing path from remote executors """ resolve_symlinks() + fix_bazel_paths() def parse(): From f038f516d4730e3b33f3499d09adb0da69721645 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Fri, 29 Aug 2025 09:06:07 +0200 Subject: [PATCH 15/15] Change regex for remote jobs --- src/codechecker_script.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/codechecker_script.py b/src/codechecker_script.py index 5ea868be..13cb0edf 100644 --- a/src/codechecker_script.py +++ b/src/codechecker_script.py @@ -27,16 +27,17 @@ CODECHECKER_ENV = "{codechecker_env}" COMPILE_COMMANDS = "{compile_commands}" -START_PATH = r"\/(?:(?!\.\s+)\S)+" BAZEL_PATHS = { - r"\/sandbox\/processwrapper-sandbox\/\S*\/execroot\/": "/execroot/", - START_PATH + r"\/worker\/build\/[0-9a-fA-F]{16}\/root\/": "", - START_PATH + r"\/[0-9a-fA-F]{32}\/execroot\/": "", - # Running bazel with --spawn_strategy=processwrapper-sandbox - # results in its path being wrongly replaced! - # In that case the 1st then 3rd regex will match and leave the path in a - # half baked state, this regex just finishes the job - r"bazel_codechecker\/": "", + # Running with Build Barn produces path output like: + # /worker/build/b301eed7f2bf2fd8/root/local_path.cc + # By removing the part until bin we have resolved the path to a local file + r"\/worker\/build\/[a-z0-9]{16}\/root\/": "", + # Sometimes the previous part is followed by this, before the local_path: + r"bazel-out\/k8-fastbuild\/bin\/": "", + # Virtual includes seems to be just a folder between the package and the + # folder containing the header files, like this: + # /path/to/package/_virtual_include/folder_with_headers/header.h + r"\/_virtual_includes\/" : "/", }