From a51bd7b7297c00c0c143eac8a85d3c76aa3549b1 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 25 Apr 2025 18:26:16 +0200 Subject: [PATCH 1/2] Unit test class for config checks Unit tests to check whether a configuration is accepted by config checks done when compilin the library. The framework provides a class and some smoke tests. It's up to the consuming branches to provide test cases based on branch-specific knowledge, and to arrange to run the tests in the CI. Signed-off-by: Gilles Peskine --- .../unittest_config_checks.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 scripts/mbedtls_framework/unittest_config_checks.py diff --git a/scripts/mbedtls_framework/unittest_config_checks.py b/scripts/mbedtls_framework/unittest_config_checks.py new file mode 100644 index 000000000..112b39808 --- /dev/null +++ b/scripts/mbedtls_framework/unittest_config_checks.py @@ -0,0 +1,171 @@ +"""Test the configuration checks that reject some bad compile-time configs. + +This tests the output of ``generate_config_checks.py``. +This can also let us verify what we enforce in the manually written +checks in ``_check_config.h``. +""" + +## Copyright The Mbed TLS Contributors +## SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import os +import subprocess +import sys +import unittest +from typing import List, Optional, Pattern, Union + + +class TestConfigChecks(unittest.TestCase): + """Unit tests for checks performed via ``_config.c``. + + This can test the code generated by `config_checks_generator`, + as well as manually written checks in `check_config.h`. + """ + + # Set this to the path to the source file containing the config checks. + PROJECT_CONFIG_C = None #type: Optional[str] + + # Project-specific include directories (in addition to /include) + PROJECT_SPECIFIC_INCLUDE_DIRECTORIES = [] #type: List[str] + + # Increase the length of strings that assertion failures are willing to + # print. This is useful for failures where the preprocessor has a lot + # to say. + maxDiff = 9999 + + def user_config_file_name(self, variant: str) -> str: + """Construct a unique temporary file name for a user config header.""" + name = os.path.splitext(os.path.basename(sys.argv[0]))[0] + pid = str(os.getpid()) + oid = str(id(self)) + return f'tmp-user_config_{variant}-{name}-{pid}-{oid}.h' + + def write_user_config(self, variant: str, content: Optional[str]) -> Optional[str]: + """Write a user configuration file with the given content. + + If content is None, ensure the file does not exist. + + Return None if content is none, otherwise return the file name. + """ + file_name = self.user_config_file_name(variant) + if content is None: + if os.path.exists(file_name): + os.remove(file_name) + return None + if content and not content.endswith('\n'): + content += '\n' + with open(file_name, 'w', encoding='ascii') as out: + out.write(content) + return file_name + + def run_with_config_files(self, + crypto_user_config_file: Optional[str], + mbedtls_user_config_file: Optional[str], + extra_options: List[str], + ) -> subprocess.CompletedProcess: + """Run cpp with the given user configuration files. + + Return the CompletedProcess object capturing the return code, + stdout and stderr. + """ + cmd = ['cpp'] + if crypto_user_config_file is not None: + cmd.append(f'-DTF_PSA_CRYPTO_USER_CONFIG_FILE="{crypto_user_config_file}"') + if mbedtls_user_config_file is not None: + cmd.append(f'-DMBEDTLS_USER_CONFIG_FILE="{mbedtls_user_config_file}"') + cmd += extra_options + assert self.PROJECT_CONFIG_C is not None + cmd += ['-I' + dir for dir in self.PROJECT_SPECIFIC_INCLUDE_DIRECTORIES] + cmd += ['-Iinclude', + '-I.', + '-I' + os.path.dirname(self.PROJECT_CONFIG_C)] + cmd.append(self.PROJECT_CONFIG_C) + return subprocess.run(cmd, + check=False, + encoding='ascii', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + def run_with_config(self, + crypto_user_config: Optional[str], + mbedtls_user_config: Optional[str] = None, + extra_options: Optional[List[str]] = None, + ) -> subprocess.CompletedProcess: + """Run cpp with the given content for user configuration files. + + Return the CompletedProcess object capturing the return code, + stdout and stderr. + """ + if extra_options is None: + extra_options = [] + crypto_user_config_file = None + mbedtls_user_config_file = None + try: + # Create temporary files without using tempfile because: + # 1. Before Python 3.12, tempfile.NamedTemporaryFile does + # not have good support for allowing an external program + # to access the file on Windows. + # 2. With a tempfile-provided context, it's awkward to not + # create a file optionally (we only do it when xxx_user_config + # is not None). + crypto_user_config_file = \ + self.write_user_config('crypto', crypto_user_config) + mbedtls_user_config_file = \ + self.write_user_config('mbedtls', mbedtls_user_config) + cp = self.run_with_config_files(crypto_user_config_file, + mbedtls_user_config_file, + extra_options) + return cp + finally: + if crypto_user_config_file is not None and \ + os.path.exists(crypto_user_config_file): + os.remove(crypto_user_config_file) + if mbedtls_user_config_file is not None and \ + os.path.exists(mbedtls_user_config_file): + os.remove(mbedtls_user_config_file) + + def good_case(self, + crypto_user_config: Optional[str], + mbedtls_user_config: Optional[str] = None, + extra_options: Optional[List[str]] = None, + ) -> None: + """Run cpp with the given user config(s). Expect no error. + + Pass extra_options on the command line of cpp. + """ + cp = self.run_with_config(crypto_user_config, mbedtls_user_config, + extra_options=extra_options) + # Assert the error text before the status. That way, if it fails, + # we see the unexpected error messages in the test log. + self.assertEqual(cp.stderr, '') + self.assertEqual(cp.returncode, 0) + + def bad_case(self, + crypto_user_config: Optional[str], + mbedtls_user_config: Optional[str] = None, + error: Optional[Union[str, Pattern]] = None, + extra_options: Optional[List[str]] = None, + ) -> None: + """Run cpp with the given user config(s). Expect errors. + + Pass extra_options on the command line of cpp. + + If error is given, the standard error from cpp must match this regex. + """ + cp = self.run_with_config(crypto_user_config, mbedtls_user_config, + extra_options=extra_options) + if error is not None: + # Assert the error text before the status. That way, if it fails, + # we see the unexpected error messages in the test log. + self.assertRegex(cp.stderr, error) + self.assertGreater(cp.returncode, 0) + self.assertLess(cp.returncode, 126) + + # Nominal case, run first + def test_01_nominal(self) -> None: + self.good_case(None) + + # Trivial error case, run second + def test_02_error(self) -> None: + self.bad_case('#error "Bad crypto configuration"', + error='"Bad crypto configuration"') From bc7bc3699370e6832b2f145648fe3a64ad0fa650 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 16 Jul 2025 22:22:09 +0200 Subject: [PATCH 2/2] unittest_config_checks debugging facility Set the environment variable `UNITTEST_CONFIG_CHECKS_DEBUG` to dump the preprocessor output to a file if a test case fails. Signed-off-by: Gilles Peskine --- .../unittest_config_checks.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/mbedtls_framework/unittest_config_checks.py b/scripts/mbedtls_framework/unittest_config_checks.py index 112b39808..ba8fc8f98 100644 --- a/scripts/mbedtls_framework/unittest_config_checks.py +++ b/scripts/mbedtls_framework/unittest_config_checks.py @@ -33,6 +33,24 @@ class TestConfigChecks(unittest.TestCase): # to say. maxDiff = 9999 + def setUp(self) -> None: + self.cpp_output = None #type: Optional[str] + + def tearDown(self) -> None: + """Log the preprocessor output to a file, if available and desired. + + This is intended for debugging. It only happens if the environment + variable UNITTEST_CONFIG_CHECKS_DEBUG is non-empty. + """ + if os.getenv('UNITTEST_CONFIG_CHECKS_DEBUG'): + # We set self.cpp_output to the preprocessor output before + # asserting, and set it to None if all the assertions pass. + if self.cpp_output is not None: + basename = os.path.splitext(os.path.basename(sys.argv[0]))[0] + filename = f'{basename}.{self._testMethodName}.out.txt' + with open(filename, 'w') as out: + out.write(self.cpp_output) + def user_config_file_name(self, variant: str) -> str: """Construct a unique temporary file name for a user config header.""" name = os.path.splitext(os.path.basename(sys.argv[0]))[0] @@ -69,6 +87,8 @@ def run_with_config_files(self, stdout and stderr. """ cmd = ['cpp'] + if os.getenv('UNITTEST_CONFIG_CHECKS_DEBUG'): + cmd += ['-dD'] if crypto_user_config_file is not None: cmd.append(f'-DTF_PSA_CRYPTO_USER_CONFIG_FILE="{crypto_user_config_file}"') if mbedtls_user_config_file is not None: @@ -137,8 +157,10 @@ def good_case(self, extra_options=extra_options) # Assert the error text before the status. That way, if it fails, # we see the unexpected error messages in the test log. + self.cpp_output = cp.stdout self.assertEqual(cp.stderr, '') self.assertEqual(cp.returncode, 0) + self.cpp_output = None def bad_case(self, crypto_user_config: Optional[str], @@ -154,12 +176,14 @@ def bad_case(self, """ cp = self.run_with_config(crypto_user_config, mbedtls_user_config, extra_options=extra_options) + self.cpp_output = cp.stdout if error is not None: # Assert the error text before the status. That way, if it fails, # we see the unexpected error messages in the test log. self.assertRegex(cp.stderr, error) self.assertGreater(cp.returncode, 0) self.assertLess(cp.returncode, 126) + self.cpp_output = None # Nominal case, run first def test_01_nominal(self) -> None: