From 8e0fac5f61dd03ebd04db9a91d47440d18495025 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Wed, 3 Sep 2025 14:47:28 +0200 Subject: [PATCH 01/13] Remove todo --- .github/workflows/foss.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/foss.yaml b/.github/workflows/foss.yaml index 3a9c0e3f..5808108c 100644 --- a/.github/workflows/foss.yaml +++ b/.github/workflows/foss.yaml @@ -25,7 +25,6 @@ concurrency: jobs: # TODO: Generalize to handle multiple projects - # TODO: Add script to run tests locally foss_ubuntu_test: name: "Test rules on FOSS project: yaml-cpp" runs-on: ubuntu-24.04 From e6e0dd05f280884317795eb7556453a1d1f6cec4 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Wed, 3 Sep 2025 14:47:55 +0200 Subject: [PATCH 02/13] Add Local foss test runner --- test/foss/__init__.py | 0 test/foss/pytest.ini | 4 ++ test/foss/test_foss.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 test/foss/__init__.py create mode 100644 test/foss/pytest.ini create mode 100644 test/foss/test_foss.py diff --git a/test/foss/__init__.py b/test/foss/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/foss/pytest.ini b/test/foss/pytest.ini new file mode 100644 index 00000000..665fb058 --- /dev/null +++ b/test/foss/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +# PyTest should not recurse into the project directories, they may contain +# other unit tests. +norecursedirs = * diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py new file mode 100644 index 00000000..0ea33aad --- /dev/null +++ b/test/foss/test_foss.py @@ -0,0 +1,94 @@ +import logging +import shlex +import subprocess +import sys +import unittest +import os + +ROOT_DIR = "./" + +NOT_PROJECT_FOLDERS = ["templates", "__pycache__", ".pytest_cache"] + + +def get_dynamic_test_dirs(): + dirs = [] + for entry in os.listdir(ROOT_DIR): + full_path = os.path.join(ROOT_DIR, entry) + if os.path.isdir(full_path) and entry not in NOT_PROJECT_FOLDERS: + dirs.append(entry) + return dirs + + +PROJECT_DIRS = get_dynamic_test_dirs() + + +# This will contain the generated tests. +class DynamicFOSSTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Configure logging before running tests""" + # Enable debug logs for tests if "super verbose" flag is provided + if "-vvv" in sys.argv: + logging.basicConfig( + level=logging.DEBUG, format="[TEST] %(levelname)5s: %(message)s" + ) + + @classmethod + def run_command( + self, cmd: str, working_dir: str = None + ) -> tuple[int, str, str]: + """ + Run shell command. + returns: + - exit code + - stdout + - stderr + """ + logging.debug("Running: %s", cmd) + commands = shlex.split(cmd) + with subprocess.Popen( + commands, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=working_dir, + ) as process: + stdout, stderr = process.communicate() + return ( + process.returncode, + f"stdout: {stdout.decode('utf-8')}", + f"stderr: {stderr.decode('utf-8')}", + ) + + +# Dynamically add a test method for each project +for dir_name in PROJECT_DIRS: + + def create_test_method(directory_name): + def test_runner(self): + project_root = os.path.join(ROOT_DIR, directory_name) + + self.assertTrue( + os.path.exists(os.path.join(project_root, "init.sh")), + f"Missing 'init.sh' in {directory_name}", + ) + project_working_dir = os.path.join(project_root, "test-proj") + if not os.path.exists(project_working_dir): + ret, _, _ = self.run_command("sh init.sh", project_root) + self.assertTrue(os.path.exists(project_working_dir)) + ret, _, _ = self.run_command( + "bazel build :codechecker_test", project_working_dir + ) + self.assertEqual(ret, 0) + ret, _, _ = self.run_command( + "bazel build :code_checker_test", project_working_dir + ) + self.assertEqual(ret, 0) + + return test_runner + + test_name = f"test_{dir_name}" + setattr(DynamicFOSSTest, test_name, create_test_method(dir_name)) + +if __name__ == "__main__": + unittest.main() From 2a213edaf2ac1b15107320c7cb08f4e402dc5b79 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Wed, 3 Sep 2025 14:50:20 +0200 Subject: [PATCH 03/13] Update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index eece9fdc..089ff5f0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ __pycache__ # Ignore Python virtual environment venv + +# Ignore FOSS project clone dirs (all directories two layer deep in foss) +/test/foss/*/*/ From 42c55e04b0ef9e4a1aac689bd8476231153c2753 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Wed, 3 Sep 2025 14:56:05 +0200 Subject: [PATCH 04/13] Add license --- test/foss/__init__.py | 13 +++++++++++++ test/foss/test_foss.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/test/foss/__init__.py b/test/foss/__init__.py index e69de29b..78bab5f1 100644 --- a/test/foss/__init__.py +++ b/test/foss/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2023 Ericsson AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 0ea33aad..451832fb 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -1,3 +1,17 @@ +# Copyright 2023 Ericsson AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import shlex import subprocess From 35c67e8e001834324e5a19406dba382218d50f1a Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 8 Sep 2025 10:21:25 +0200 Subject: [PATCH 05/13] Add verbose comments to test_foss.py --- test/foss/test_foss.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 451832fb..41690e45 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -20,7 +20,6 @@ import os ROOT_DIR = "./" - NOT_PROJECT_FOLDERS = ["templates", "__pycache__", ".pytest_cache"] @@ -37,6 +36,9 @@ def get_dynamic_test_dirs(): # This will contain the generated tests. +# I have not used the common lib from unit test, because it would +# greatly increase the difficulty of the implementation, and this way the +# two test aren't bound to each other class DynamicFOSSTest(unittest.TestCase): @classmethod def setUpClass(cls): @@ -76,6 +78,7 @@ def run_command( # Dynamically add a test method for each project +# For each project directory it adds a new test function to the class for dir_name in PROJECT_DIRS: def create_test_method(directory_name): From 50175fea973572a3f9dc28607d0bfc10396fdcc6 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 9 Sep 2025 14:21:57 +0200 Subject: [PATCH 06/13] Move pytest.ini to the root --- pytest.ini | 10 ++++++++++ test/foss/pytest.ini | 4 ---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 pytest.ini delete mode 100644 test/foss/pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..0644f898 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,10 @@ +[pytest] +# PyTest should not recurse into the project directories, they may contain +# other unit tests. +testpaths = test/foss +norecursedirs = + test/foss/*/test-proj + # This is why the template dir is named different in `unit` and `foss` + templates + __pycache__ + .pytest_cache diff --git a/test/foss/pytest.ini b/test/foss/pytest.ini deleted file mode 100644 index 665fb058..00000000 --- a/test/foss/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -# PyTest should not recurse into the project directories, they may contain -# other unit tests. -norecursedirs = * From 52f4f2c4add02f6efd23ca3438e52f1e9b6458b1 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 9 Sep 2025 14:27:09 +0200 Subject: [PATCH 07/13] Fixup test_foss.py --- test/foss/test_foss.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 41690e45..531de4e4 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -19,11 +19,11 @@ import unittest import os -ROOT_DIR = "./" +ROOT_DIR = f"{os.path.dirname(os.path.abspath(__file__))}/" NOT_PROJECT_FOLDERS = ["templates", "__pycache__", ".pytest_cache"] -def get_dynamic_test_dirs(): +def get_test_dirs() -> list[str]: dirs = [] for entry in os.listdir(ROOT_DIR): full_path = os.path.join(ROOT_DIR, entry) @@ -32,26 +32,30 @@ def get_dynamic_test_dirs(): return dirs -PROJECT_DIRS = get_dynamic_test_dirs() +PROJECT_DIRS = get_test_dirs() # This will contain the generated tests. # I have not used the common lib from unit test, because it would # greatly increase the difficulty of the implementation, and this way the # two test aren't bound to each other -class DynamicFOSSTest(unittest.TestCase): +class FOSSTestCollector(unittest.TestCase): @classmethod def setUpClass(cls): """Configure logging before running tests""" + # Change working directory to test/foss + os.chdir(os.path.dirname(os.path.abspath(__file__))) # Enable debug logs for tests if "super verbose" flag is provided if "-vvv" in sys.argv: logging.basicConfig( level=logging.DEBUG, format="[TEST] %(levelname)5s: %(message)s" ) + # I added this here because its much easier than including the base class + # From the unit tests common lib @classmethod def run_command( - self, cmd: str, working_dir: str = None + self, cmd: str, working_dir: str|None = None ) -> tuple[int, str, str]: """ Run shell command. @@ -105,7 +109,7 @@ def test_runner(self): return test_runner test_name = f"test_{dir_name}" - setattr(DynamicFOSSTest, test_name, create_test_method(dir_name)) + setattr(FOSSTestCollector, test_name, create_test_method(dir_name)) if __name__ == "__main__": unittest.main() From 102be15d14e1cb55407f8370c59222821d5ea123 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 9 Sep 2025 14:40:18 +0200 Subject: [PATCH 08/13] Refine with comments, minimal refactoring --- test/foss/test_foss.py | 63 ++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 531de4e4..6ae2c228 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -18,6 +18,7 @@ import sys import unittest import os +from types import FunctionType ROOT_DIR = f"{os.path.dirname(os.path.abspath(__file__))}/" NOT_PROJECT_FOLDERS = ["templates", "__pycache__", ".pytest_cache"] @@ -80,36 +81,38 @@ def run_command( f"stderr: {stderr.decode('utf-8')}", ) - -# Dynamically add a test method for each project -# For each project directory it adds a new test function to the class -for dir_name in PROJECT_DIRS: - - def create_test_method(directory_name): - def test_runner(self): - project_root = os.path.join(ROOT_DIR, directory_name) - - self.assertTrue( - os.path.exists(os.path.join(project_root, "init.sh")), - f"Missing 'init.sh' in {directory_name}", - ) - project_working_dir = os.path.join(project_root, "test-proj") - if not os.path.exists(project_working_dir): - ret, _, _ = self.run_command("sh init.sh", project_root) - self.assertTrue(os.path.exists(project_working_dir)) - ret, _, _ = self.run_command( - "bazel build :codechecker_test", project_working_dir - ) - self.assertEqual(ret, 0) - ret, _, _ = self.run_command( - "bazel build :code_checker_test", project_working_dir - ) - self.assertEqual(ret, 0) - - return test_runner - - test_name = f"test_{dir_name}" - setattr(FOSSTestCollector, test_name, create_test_method(dir_name)) +# Creates test functions with the parameter: directory_name. Based on: +# https://eli.thegreenplace.net/2014/04/02/dynamically-generating-python-test-cases +def create_test_method(directory_name: str) -> FunctionType: + """ + Returns a function pointer that points to a function for the given directory + """ + def test_runner(self) -> None: + project_root = os.path.join(ROOT_DIR, directory_name) + + self.assertTrue( + os.path.exists(os.path.join(project_root, "init.sh")), + f"Missing 'init.sh' in {directory_name}", + ) + project_working_dir = os.path.join(project_root, "test-proj") + if not os.path.exists(project_working_dir): + ret, _, _ = self.run_command("sh init.sh", project_root) + self.assertTrue(os.path.exists(project_working_dir)) + ret, _, _ = self.run_command( + "bazel build :codechecker_test", project_working_dir + ) + self.assertEqual(ret, 0) + ret, _, _ = self.run_command( + "bazel build :code_checker_test", project_working_dir + ) + self.assertEqual(ret, 0) + + return test_runner if __name__ == "__main__": + # Dynamically add a test method for each project + # For each project directory it adds a new test function to the class + for dir_name in PROJECT_DIRS: + test_name = f"test_{dir_name}" + setattr(FOSSTestCollector, test_name, create_test_method(dir_name)) unittest.main() From 87a0411f5e0aa7c47fe20f38ad91a71ffd71c169 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 9 Sep 2025 15:15:49 +0200 Subject: [PATCH 09/13] Remove python 3.10 specific syntax --- test/foss/test_foss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 6ae2c228..298a1e01 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -56,7 +56,7 @@ def setUpClass(cls): # From the unit tests common lib @classmethod def run_command( - self, cmd: str, working_dir: str|None = None + self, cmd: str, working_dir: str = None ) -> tuple[int, str, str]: """ Run shell command. From 38fca3294b6a9d3a4fd2c7ce1aca4d16929ec16d Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Tue, 9 Sep 2025 15:20:33 +0200 Subject: [PATCH 10/13] Move dynamin test creation --- test/foss/test_foss.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 298a1e01..145b4e35 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -109,10 +109,12 @@ def test_runner(self) -> None: return test_runner +# Dynamically add a test method for each project +# For each project directory it adds a new test function to the class +# This must be outside of the __main__ if, pytest doesn't run it that way +for dir_name in PROJECT_DIRS: + test_name = f"test_{dir_name}" + setattr(FOSSTestCollector, test_name, create_test_method(dir_name)) + if __name__ == "__main__": - # Dynamically add a test method for each project - # For each project directory it adds a new test function to the class - for dir_name in PROJECT_DIRS: - test_name = f"test_{dir_name}" - setattr(FOSSTestCollector, test_name, create_test_method(dir_name)) unittest.main() From 686a8988723c33d428b27a6139655413f944ff8e Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Wed, 10 Sep 2025 08:45:30 +0200 Subject: [PATCH 11/13] Move common lib to test, use it in FOSS tests --- test/__init__.py | 24 ++++++++++++++ test/{unit => }/common/__init__.py | 0 test/{unit => }/common/base.py | 0 test/foss/test_foss.py | 51 ++++-------------------------- test/unit/__init__.py | 24 -------------- 5 files changed, 31 insertions(+), 68 deletions(-) create mode 100644 test/__init__.py rename test/{unit => }/common/__init__.py (100%) rename test/{unit => }/common/base.py (100%) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e6deb66a --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2023 Ericsson AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Setup module paths and environment variables for the functional tests. +""" + +import os +import sys + +# Allow relative imports within the test project to work as expected +# Without it no module (test) would be able to import the common library +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)))) diff --git a/test/unit/common/__init__.py b/test/common/__init__.py similarity index 100% rename from test/unit/common/__init__.py rename to test/common/__init__.py diff --git a/test/unit/common/base.py b/test/common/base.py similarity index 100% rename from test/unit/common/base.py rename to test/common/base.py diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 145b4e35..dfe9f37d 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -13,12 +13,11 @@ # limitations under the License. import logging -import shlex -import subprocess import sys import unittest import os from types import FunctionType +from common.base import TestBase ROOT_DIR = f"{os.path.dirname(os.path.abspath(__file__))}/" NOT_PROJECT_FOLDERS = ["templates", "__pycache__", ".pytest_cache"] @@ -37,49 +36,13 @@ def get_test_dirs() -> list[str]: # This will contain the generated tests. -# I have not used the common lib from unit test, because it would -# greatly increase the difficulty of the implementation, and this way the -# two test aren't bound to each other -class FOSSTestCollector(unittest.TestCase): - @classmethod - def setUpClass(cls): - """Configure logging before running tests""" - # Change working directory to test/foss - os.chdir(os.path.dirname(os.path.abspath(__file__))) - # Enable debug logs for tests if "super verbose" flag is provided - if "-vvv" in sys.argv: - logging.basicConfig( - level=logging.DEBUG, format="[TEST] %(levelname)5s: %(message)s" - ) +class FOSSTestCollector(TestBase): + # Set working directory + __test_path__ = os.path.dirname(os.path.abspath(__file__)) + # These are irrelevant for these kind of tests + BAZEL_BIN_DIR = os.path.join("") + BAZEL_TESTLOGS_DIR = os.path.join("") - # I added this here because its much easier than including the base class - # From the unit tests common lib - @classmethod - def run_command( - self, cmd: str, working_dir: str = None - ) -> tuple[int, str, str]: - """ - Run shell command. - returns: - - exit code - - stdout - - stderr - """ - logging.debug("Running: %s", cmd) - commands = shlex.split(cmd) - with subprocess.Popen( - commands, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=working_dir, - ) as process: - stdout, stderr = process.communicate() - return ( - process.returncode, - f"stdout: {stdout.decode('utf-8')}", - f"stderr: {stderr.decode('utf-8')}", - ) # Creates test functions with the parameter: directory_name. Based on: # https://eli.thegreenplace.net/2014/04/02/dynamically-generating-python-test-cases diff --git a/test/unit/__init__.py b/test/unit/__init__.py index e6deb66a..e69de29b 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -1,24 +0,0 @@ -# Copyright 2023 Ericsson AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Setup module paths and environment variables for the functional tests. -""" - -import os -import sys - -# Allow relative imports within the test project to work as expected -# Without it no module (test) would be able to import the common library -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)))) From 7e762327f34756f03b8c59f229060524bda21f2d Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 15 Sep 2025 09:54:18 +0200 Subject: [PATCH 12/13] Specify which dir to ignore .gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 089ff5f0..8bdb1869 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,5 @@ __pycache__ # Ignore Python virtual environment venv -# Ignore FOSS project clone dirs (all directories two layer deep in foss) -/test/foss/*/*/ +# Ignore FOSS project clone dirs +/test/foss/*/test-proj/ From 8697e4522fbc848dda55da2ff0a47c4210769be7 Mon Sep 17 00:00:00 2001 From: "F.Tibor" Date: Mon, 15 Sep 2025 10:28:50 +0200 Subject: [PATCH 13/13] Add helpful message to failing test --- test/foss/test_foss.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index dfe9f37d..02ca7881 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -55,7 +55,8 @@ def test_runner(self) -> None: self.assertTrue( os.path.exists(os.path.join(project_root, "init.sh")), - f"Missing 'init.sh' in {directory_name}", + f"Missing 'init.sh' in {directory_name}\n" + \ + "Please consult with the README on how to add a new FOSS project", ) project_working_dir = os.path.join(project_root, "test-proj") if not os.path.exists(project_working_dir):