From 3008a4a7f68d0972c3aede0950dbdc131b15f6b8 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 11 Sep 2025 17:15:15 +0200 Subject: [PATCH 01/14] Put the command line entry point in a function Signed-off-by: Gilles Peskine --- scripts/test_psa_compliance.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/test_psa_compliance.py b/scripts/test_psa_compliance.py index 1b51f0fe1..8e98a5190 100755 --- a/scripts/test_psa_compliance.py +++ b/scripts/test_psa_compliance.py @@ -32,7 +32,8 @@ PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC' #pylint: disable=too-many-branches,too-many-statements,too-many-locals -def main(library_build_dir: str, expected_failures: List[int]): +def test_compliance(library_build_dir: str, expected_failures: List[int]): + """Check out and run compliance tests.""" root_dir = os.getcwd() install_dir = Path(library_build_dir + "/install_dir").resolve() tmp_env = os.environ @@ -134,8 +135,9 @@ def main(library_build_dir: str, expected_failures: List[int]): finally: os.chdir(root_dir) -if __name__ == '__main__': - BUILD_DIR = 'out_of_source_build' +def main() -> None: + """Command line entry point.""" + build_dir = 'out_of_source_build' # pylint: disable=invalid-name parser = argparse.ArgumentParser() @@ -148,11 +150,14 @@ def main(library_build_dir: str, expected_failures: List[int]): args = parser.parse_args() if args.build_dir is not None: - BUILD_DIR = args.build_dir[0] + build_dir = args.build_dir[0] if args.expected_failures is not None: expected_failures_list = [int(i) for i in args.expected_failures] else: expected_failures_list = EXPECTED_FAILURES - sys.exit(main(BUILD_DIR, expected_failures_list)) + sys.exit(test_compliance(build_dir, expected_failures_list)) + +if __name__ == '__main__': + main() From 26cb1a173b648c646227852fb5b535f59882e44b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 11 Sep 2025 17:55:21 +0200 Subject: [PATCH 02/14] Move the bulk of test_psa_compliance.py to a module This will allow consuming branches to each have their executable entry point. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 159 ++++++++++++++++++++ scripts/test_psa_compliance.py | 158 +------------------ 2 files changed, 163 insertions(+), 154 deletions(-) create mode 100644 scripts/mbedtls_framework/psa_compliance.py diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py new file mode 100644 index 000000000..14e37e78d --- /dev/null +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -0,0 +1,159 @@ +"""Run the PSA Crypto API compliance test suite. +Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, +then compile and run the test suite. The clone is stored at /psa-arch-tests. +Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test +number - are ignored, while unexpected failures AND successes are reported as errors, to help +keep the list of known defects as up to date as possible. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import argparse +import os +import re +import shutil +import subprocess +import sys +from typing import List +from pathlib import Path + +from . import build_tree + +# PSA Compliance tests we expect to fail due to known defects in Mbed TLS / +# TF-PSA-Crypto (or the test suite). +# The test numbers correspond to the numbers used by the console output of the test suite. +# Test number 2xx corresponds to the files in the folder +# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx +EXPECTED_FAILURES = [] # type: List[int] + +PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git' +PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC' + +#pylint: disable=too-many-branches,too-many-statements,too-many-locals +def test_compliance(library_build_dir: str, expected_failures: List[int]): + """Check out and run compliance tests.""" + root_dir = os.getcwd() + install_dir = Path(library_build_dir + "/install_dir").resolve() + tmp_env = os.environ + tmp_env['CC'] = 'gcc' + subprocess.check_call(['cmake', '.', '-GUnix Makefiles', + '-B' + library_build_dir, + '-DCMAKE_INSTALL_PREFIX=' + str(install_dir)], + env=tmp_env) + subprocess.check_call(['cmake', '--build', library_build_dir, '--target', 'install']) + + if build_tree.is_mbedtls_3_6(): + crypto_library_path = install_dir.joinpath("lib/libmbedcrypto.a") + else: + crypto_library_path = install_dir.joinpath("lib/libtfpsacrypto.a") + + psa_arch_tests_dir = 'psa-arch-tests' + os.makedirs(psa_arch_tests_dir, exist_ok=True) + try: + os.chdir(psa_arch_tests_dir) + + # Reuse existing local clone + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) + + build_dir = 'api-tests/build' + try: + shutil.rmtree(build_dir) + except FileNotFoundError: + pass + os.mkdir(build_dir) + os.chdir(build_dir) + + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '..', + '-GUnix Makefiles', + '-DTARGET=tgt_dev_apis_stdc', + '-DTOOLCHAIN=HOST_GCC', + '-DSUITE=CRYPTO', + '-DPSA_CRYPTO_LIB_FILENAME={}'.format(str(crypto_library_path)), + '-DPSA_INCLUDE_PATHS=' + str(install_dir.joinpath("include")) + ]) + + subprocess.check_call(['cmake', '--build', '.']) + + proc = subprocess.Popen(['./psa-arch-tests-crypto'], + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + + test_re = re.compile( + '^TEST: (?P[0-9]*)|' + '^TEST RESULT: (?PFAILED|PASSED)' + ) + test = -1 + unexpected_successes = expected_failures.copy() + expected_failures.clear() + unexpected_failures = [] # type: List[int] + if proc.stdout is None: + return 1 + + for line in proc.stdout: + print(line, end='') + match = test_re.match(line) + if match is not None: + groupdict = match.groupdict() + test_num = groupdict['test_num'] + if test_num is not None: + test = int(test_num) + elif groupdict['test_result'] == 'FAILED': + try: + unexpected_successes.remove(test) + expected_failures.append(test) + print('Expected failure, ignoring') + except KeyError: + unexpected_failures.append(test) + print('ERROR: Unexpected failure') + elif test in unexpected_successes: + print('ERROR: Unexpected success') + proc.wait() + + print() + print('***** test_psa_compliance.py report ******') + print() + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) + print() + if unexpected_successes or unexpected_failures: + if unexpected_successes: + print('Unexpected successes encountered.') + print('Please remove the corresponding tests from ' + 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print() + print('FAILED') + return 1 + else: + print('SUCCESS') + return 0 + finally: + os.chdir(root_dir) + +def main() -> None: + """Command line entry point.""" + build_dir = 'out_of_source_build' + + # pylint: disable=invalid-name + parser = argparse.ArgumentParser() + parser.add_argument('--build-dir', nargs=1, + help='path to Mbed TLS / TF-PSA-Crypto build directory') + parser.add_argument('--expected-failures', nargs='+', + help='''set the list of test codes which are expected to fail + from the command line. If omitted the list given by + EXPECTED_FAILURES (inside the script) is used.''') + args = parser.parse_args() + + if args.build_dir is not None: + build_dir = args.build_dir[0] + + if args.expected_failures is not None: + expected_failures_list = [int(i) for i in args.expected_failures] + else: + expected_failures_list = EXPECTED_FAILURES + + sys.exit(test_compliance(build_dir, expected_failures_list)) diff --git a/scripts/test_psa_compliance.py b/scripts/test_psa_compliance.py index 8e98a5190..5eaf07163 100755 --- a/scripts/test_psa_compliance.py +++ b/scripts/test_psa_compliance.py @@ -1,163 +1,13 @@ #!/usr/bin/env python3 """Run the PSA Crypto API compliance test suite. -Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, -then compile and run the test suite. The clone is stored at /psa-arch-tests. -Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test -number - are ignored, while unexpected failures AND successes are reported as errors, to help -keep the list of known defects as up to date as possible. + +Transitional wrapper to facilitate the migration of consuming branches. """ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later -import argparse -import os -import re -import shutil -import subprocess -import sys -from typing import List -from pathlib import Path - -from mbedtls_framework import build_tree - -# PSA Compliance tests we expect to fail due to known defects in Mbed TLS / -# TF-PSA-Crypto (or the test suite). -# The test numbers correspond to the numbers used by the console output of the test suite. -# Test number 2xx corresponds to the files in the folder -# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx -EXPECTED_FAILURES = [] # type: List[int] - -PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git' -PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC' - -#pylint: disable=too-many-branches,too-many-statements,too-many-locals -def test_compliance(library_build_dir: str, expected_failures: List[int]): - """Check out and run compliance tests.""" - root_dir = os.getcwd() - install_dir = Path(library_build_dir + "/install_dir").resolve() - tmp_env = os.environ - tmp_env['CC'] = 'gcc' - subprocess.check_call(['cmake', '.', '-GUnix Makefiles', - '-B' + library_build_dir, - '-DCMAKE_INSTALL_PREFIX=' + str(install_dir)], - env=tmp_env) - subprocess.check_call(['cmake', '--build', library_build_dir, '--target', 'install']) - - if build_tree.is_mbedtls_3_6(): - crypto_library_path = install_dir.joinpath("lib/libmbedcrypto.a") - else: - crypto_library_path = install_dir.joinpath("lib/libtfpsacrypto.a") - - psa_arch_tests_dir = 'psa-arch-tests' - os.makedirs(psa_arch_tests_dir, exist_ok=True) - try: - os.chdir(psa_arch_tests_dir) - - # Reuse existing local clone - subprocess.check_call(['git', 'init']) - subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) - subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) - - build_dir = 'api-tests/build' - try: - shutil.rmtree(build_dir) - except FileNotFoundError: - pass - os.mkdir(build_dir) - os.chdir(build_dir) - - #pylint: disable=bad-continuation - subprocess.check_call([ - 'cmake', '..', - '-GUnix Makefiles', - '-DTARGET=tgt_dev_apis_stdc', - '-DTOOLCHAIN=HOST_GCC', - '-DSUITE=CRYPTO', - '-DPSA_CRYPTO_LIB_FILENAME={}'.format(str(crypto_library_path)), - '-DPSA_INCLUDE_PATHS=' + str(install_dir.joinpath("include")) - ]) - - subprocess.check_call(['cmake', '--build', '.']) - - proc = subprocess.Popen(['./psa-arch-tests-crypto'], - bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) - - test_re = re.compile( - '^TEST: (?P[0-9]*)|' - '^TEST RESULT: (?PFAILED|PASSED)' - ) - test = -1 - unexpected_successes = expected_failures.copy() - expected_failures.clear() - unexpected_failures = [] # type: List[int] - if proc.stdout is None: - return 1 - - for line in proc.stdout: - print(line, end='') - match = test_re.match(line) - if match is not None: - groupdict = match.groupdict() - test_num = groupdict['test_num'] - if test_num is not None: - test = int(test_num) - elif groupdict['test_result'] == 'FAILED': - try: - unexpected_successes.remove(test) - expected_failures.append(test) - print('Expected failure, ignoring') - except KeyError: - unexpected_failures.append(test) - print('ERROR: Unexpected failure') - elif test in unexpected_successes: - print('ERROR: Unexpected success') - proc.wait() - - print() - print('***** test_psa_compliance.py report ******') - print() - print('Expected failures:', ', '.join(str(i) for i in expected_failures)) - print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) - print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) - print() - if unexpected_successes or unexpected_failures: - if unexpected_successes: - print('Unexpected successes encountered.') - print('Please remove the corresponding tests from ' - 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') - print() - print('FAILED') - return 1 - else: - print('SUCCESS') - return 0 - finally: - os.chdir(root_dir) - -def main() -> None: - """Command line entry point.""" - build_dir = 'out_of_source_build' - - # pylint: disable=invalid-name - parser = argparse.ArgumentParser() - parser.add_argument('--build-dir', nargs=1, - help='path to Mbed TLS / TF-PSA-Crypto build directory') - parser.add_argument('--expected-failures', nargs='+', - help='''set the list of test codes which are expected to fail - from the command line. If omitted the list given by - EXPECTED_FAILURES (inside the script) is used.''') - args = parser.parse_args() - - if args.build_dir is not None: - build_dir = args.build_dir[0] - - if args.expected_failures is not None: - expected_failures_list = [int(i) for i in args.expected_failures] - else: - expected_failures_list = EXPECTED_FAILURES - - sys.exit(test_compliance(build_dir, expected_failures_list)) +from mbedtls_framework import psa_compliance if __name__ == '__main__': - main() + psa_compliance.main() From 43c6cbc9311c53668ff81d52354fbc3bd2f3bf31 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 11 Sep 2025 18:05:48 +0200 Subject: [PATCH 03/14] Move branch-dependent defaults to the calling script Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 23 +++++++++------------ scripts/test_psa_compliance.py | 13 +++++++++++- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 14e37e78d..4c2d1ce1d 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -20,18 +20,12 @@ from . import build_tree -# PSA Compliance tests we expect to fail due to known defects in Mbed TLS / -# TF-PSA-Crypto (or the test suite). -# The test numbers correspond to the numbers used by the console output of the test suite. -# Test number 2xx corresponds to the files in the folder -# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx -EXPECTED_FAILURES = [] # type: List[int] - PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git' -PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC' #pylint: disable=too-many-branches,too-many-statements,too-many-locals -def test_compliance(library_build_dir: str, expected_failures: List[int]): +def test_compliance(library_build_dir: str, + psa_arch_tests_ref: str, + expected_failures: List[int]) -> int: """Check out and run compliance tests.""" root_dir = os.getcwd() install_dir = Path(library_build_dir + "/install_dir").resolve() @@ -55,7 +49,7 @@ def test_compliance(library_build_dir: str, expected_failures: List[int]): # Reuse existing local clone subprocess.check_call(['git', 'init']) - subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, psa_arch_tests_ref]) subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) build_dir = 'api-tests/build' @@ -134,7 +128,8 @@ def test_compliance(library_build_dir: str, expected_failures: List[int]): finally: os.chdir(root_dir) -def main() -> None: +def main(psa_arch_tests_ref: str, + expected_failures: List[int] = []) -> None: """Command line entry point.""" build_dir = 'out_of_source_build' @@ -154,6 +149,8 @@ def main() -> None: if args.expected_failures is not None: expected_failures_list = [int(i) for i in args.expected_failures] else: - expected_failures_list = EXPECTED_FAILURES + expected_failures_list = expected_failures - sys.exit(test_compliance(build_dir, expected_failures_list)) + sys.exit(test_compliance(build_dir, + psa_arch_tests_ref, + expected_failures_list)) diff --git a/scripts/test_psa_compliance.py b/scripts/test_psa_compliance.py index 5eaf07163..51d6b4a7e 100755 --- a/scripts/test_psa_compliance.py +++ b/scripts/test_psa_compliance.py @@ -7,7 +7,18 @@ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +from typing import List + from mbedtls_framework import psa_compliance +PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC' + +# PSA Compliance tests we expect to fail due to known defects in Mbed TLS / +# TF-PSA-Crypto (or the test suite). +# The test numbers correspond to the numbers used by the console output of the test suite. +# Test number 2xx corresponds to the files in the folder +# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx +EXPECTED_FAILURES = [] # type: List[int] + if __name__ == '__main__': - psa_compliance.main() + psa_compliance.main(PSA_ARCH_TESTS_REF, expected_failures=EXPECTED_FAILURES) From 4ff6a49ad2fa55bc5c6e20e4a70374f990154278 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 11 Sep 2025 18:31:21 +0200 Subject: [PATCH 04/14] Support applying a patch to the compliance suite Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 27 ++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 4c2d1ce1d..5e3e8ec7d 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -25,8 +25,15 @@ #pylint: disable=too-many-branches,too-many-statements,too-many-locals def test_compliance(library_build_dir: str, psa_arch_tests_ref: str, + patch: str, expected_failures: List[int]) -> int: - """Check out and run compliance tests.""" + """Check out and run compliance tests. + + library_build_dir: path where our library will be built. + psa_arch_tests_ref: tag or sha to use for the arch-tests. + patch: patch to apply to the arch-tests with ``patch -p1``. + expected_failures: default list of expected failures. + """ root_dir = os.getcwd() install_dir = Path(library_build_dir + "/install_dir").resolve() tmp_env = os.environ @@ -50,7 +57,14 @@ def test_compliance(library_build_dir: str, # Reuse existing local clone subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, psa_arch_tests_ref]) - subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) + subprocess.check_call(['git', 'checkout', '--force', 'FETCH_HEAD']) + + if patch: + subprocess.check_call(['git', 'reset', '--hard']) + subprocess.run(['patch', '-p1'], + check=True, + encoding='utf-8', + input=patch) build_dir = 'api-tests/build' try: @@ -129,8 +143,14 @@ def test_compliance(library_build_dir: str, os.chdir(root_dir) def main(psa_arch_tests_ref: str, + patch: str = '', expected_failures: List[int] = []) -> None: - """Command line entry point.""" + """Command line entry point. + + psa_arch_tests_ref: tag or sha to use for the arch-tests. + patch: patch to apply to the arch-tests with ``patch -p1``. + expected_failures: default list of expected failures. + """ build_dir = 'out_of_source_build' # pylint: disable=invalid-name @@ -153,4 +173,5 @@ def main(psa_arch_tests_ref: str, sys.exit(test_compliance(build_dir, psa_arch_tests_ref, + patch, expected_failures_list)) From af6b1894b0a80c59bc6201326fae61b7880e1b92 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 11 Sep 2025 21:51:53 +0200 Subject: [PATCH 05/14] Teach crypto_knowledge the categories of SP800_100 and SPAKE2P algorithms Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/crypto_knowledge.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/mbedtls_framework/crypto_knowledge.py b/scripts/mbedtls_framework/crypto_knowledge.py index ebfd55cdb..f69444c16 100644 --- a/scripts/mbedtls_framework/crypto_knowledge.py +++ b/scripts/mbedtls_framework/crypto_knowledge.py @@ -355,12 +355,14 @@ def determine_head(expr: str) -> str: 'TLS12_PRF': AlgorithmCategory.KEY_DERIVATION, 'TLS12_PSK_TO_MS': AlgorithmCategory.KEY_DERIVATION, 'TLS12_ECJPAKE_TO_PMS': AlgorithmCategory.KEY_DERIVATION, + 'SP800_108': AlgorithmCategory.KEY_DERIVATION, 'PBKDF': AlgorithmCategory.KEY_DERIVATION, 'ECDH': AlgorithmCategory.KEY_AGREEMENT, 'FFDH': AlgorithmCategory.KEY_AGREEMENT, # KEY_AGREEMENT(...) is a key derivation with a key agreement component 'KEY_AGREEMENT': AlgorithmCategory.KEY_DERIVATION, 'JPAKE': AlgorithmCategory.PAKE, + 'SPAKE2P': AlgorithmCategory.PAKE, } for x in BLOCK_MAC_MODES: CATEGORY_FROM_HEAD[x] = AlgorithmCategory.MAC From 565b4c35bfe06361b86ea04b2768b91f90b4a493 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 11 Sep 2025 21:52:44 +0200 Subject: [PATCH 06/14] Skip all knowledge of SPAKE2+ keys for now Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_information.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/mbedtls_framework/psa_information.py b/scripts/mbedtls_framework/psa_information.py index f0d0e7974..cac16b5da 100644 --- a/scripts/mbedtls_framework/psa_information.py +++ b/scripts/mbedtls_framework/psa_information.py @@ -26,12 +26,18 @@ def remove_unwanted_macros( """Remove constructors that should be exckuded from systematic testing.""" # Mbed TLS does not support finite-field DSA, but 3.6 defines DSA # identifiers for historical reasons. + # Mbed TLS and TF-PSA-Crypto 1.0 do not support SPAKE2+, although + # TF-PSA-Crypto 1.0 defines SPAKE2+ identifiers to be able to build + # the psa-arch-tests compliance test suite. + # # Don't attempt to generate any related test case. # The corresponding test cases would be commented out anyway, - # but for DSA, we don't have enough support in the test scripts + # but for these types, we don't have enough support in the test scripts # to generate these test cases. constructors.key_types.discard('PSA_KEY_TYPE_DSA_KEY_PAIR') constructors.key_types.discard('PSA_KEY_TYPE_DSA_PUBLIC_KEY') + constructors.key_types.discard('PSA_KEY_TYPE_SPAKE2P_KEY_PAIR') + constructors.key_types.discard('PSA_KEY_TYPE_SPAKE2P_PUBLIC_KEY') def read_psa_interface(self) -> macro_collector.PSAMacroEnumerator: """Return the list of known key types, algorithms, etc.""" From b4035db7a1414abd898c3056f39110e5a60903ba Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 10:44:17 +0200 Subject: [PATCH 07/14] Fix [] used as a default argument value (mutable default values are dodgy) Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 5e3e8ec7d..c7bdd37e3 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -15,7 +15,7 @@ import shutil import subprocess import sys -from typing import List +from typing import List, Optional from pathlib import Path from . import build_tree @@ -144,7 +144,7 @@ def test_compliance(library_build_dir: str, def main(psa_arch_tests_ref: str, patch: str = '', - expected_failures: List[int] = []) -> None: + expected_failures: Optional[List[int]] = None) -> None: """Command line entry point. psa_arch_tests_ref: tag or sha to use for the arch-tests. @@ -166,6 +166,8 @@ def main(psa_arch_tests_ref: str, if args.build_dir is not None: build_dir = args.build_dir[0] + if expected_failures is None: + expected_failures = [] if args.expected_failures is not None: expected_failures_list = [int(i) for i in args.expected_failures] else: From 954783cb017697547b902f96227af849bbf8c757 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 11:53:41 +0200 Subject: [PATCH 08/14] macro_collector: be stricter about unusual macros When looking for constructors, do complain if we see an unusual macro with a parameter: there's a significant chance that it's something new that will require specific handling. As a consequence, we need to explicitly skip more things that are known not to be constructors. Keep ignoring macros without parameters that don't look like constructors for the types we care about. Those probably don't matter. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/macro_collector.py | 31 ++++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/mbedtls_framework/macro_collector.py b/scripts/mbedtls_framework/macro_collector.py index d68be00bd..132d59ad1 100644 --- a/scripts/mbedtls_framework/macro_collector.py +++ b/scripts/mbedtls_framework/macro_collector.py @@ -279,12 +279,27 @@ def record_algorithm_subtype(self, name: str, expansion: str) -> None: r'(.+)') _deprecated_definition_re = re.compile(r'\s*MBEDTLS_DEPRECATED') + # Macro that is a destructor, not a constructor (i.e. takes a thing as + # an argument and analyzes it, rather than constructing a thing). + _destructor_name_re = re.compile(r'.*(_GET_|_IS_)|.*_LENGTH\Z') + + # Macro that converts between things, rather than building a thing from + # scratch. + _conversion_macro_names = frozenset([ + 'PSA_KEY_TYPE_KEY_PAIR_OF_PUBLIC_KEY', + 'PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR', + 'PSA_ALG_FULL_LENGTH_MAC', + 'PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG', + 'PSA_JPAKE_EXPECTED_INPUTS', + 'PSA_JPAKE_EXPECTED_OUTPUTS', + ]) + def read_line(self, line): """Parse a C header line and record the PSA identifier it defines if any. This function analyzes lines that start with "#define PSA_" (up to non-significant whitespace) and skips all non-matching lines. """ - # pylint: disable=too-many-branches + # pylint: disable=too-many-branches,too-many-return-statements m = re.match(self._define_directive_re, line) if not m: return @@ -297,6 +312,12 @@ def read_line(self, line): # backward compatibility aliases that share # numerical values with non-deprecated values. return + if re.match(self._destructor_name_re, name): + # Not a constructor + return + if name in self._conversion_macro_names: + # Not a constructor + return if self.is_internal_name(name): # Macro only to build actual values return @@ -324,9 +345,13 @@ def read_line(self, line): self.algorithms_from_hash[name] = self.algorithm_tester(name) elif name.startswith('PSA_KEY_USAGE_') and not parameter: self.key_usage_flags.add(name) - else: - # Other macro without parameter + elif parameter is None: + # Macro with no parameter, whose name does not start with one + # of the prefixes we look for. Just ignore it. return + else: + raise Exception("Unsupported macro and parameter name: {}({})" + .format(name, parameter)) _nonascii_re = re.compile(rb'[^\x00-\x7f]+') _continued_line_re = re.compile(rb'\\\r?\n\Z') From ca98305f7f7f23a8012f0d057ec423180b7c831e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 11:56:15 +0200 Subject: [PATCH 09/14] Treat xxx_HAS_xxx as not a constructor, like xxx_IS_xxx etc. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/macro_collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_framework/macro_collector.py b/scripts/mbedtls_framework/macro_collector.py index 132d59ad1..bc10418b3 100644 --- a/scripts/mbedtls_framework/macro_collector.py +++ b/scripts/mbedtls_framework/macro_collector.py @@ -281,7 +281,7 @@ def record_algorithm_subtype(self, name: str, expansion: str) -> None: # Macro that is a destructor, not a constructor (i.e. takes a thing as # an argument and analyzes it, rather than constructing a thing). - _destructor_name_re = re.compile(r'.*(_GET_|_IS_)|.*_LENGTH\Z') + _destructor_name_re = re.compile(r'.*(_GET_|_HAS_|_IS_)|.*_LENGTH\Z') # Macro that converts between things, rather than building a thing from # scratch. @@ -476,7 +476,7 @@ def get_names(self, type_word: str) -> Set[str]: r'(PSA_((?:(?:DH|ECC|KEY)_)?[A-Z]+)_\w+)' + r'(?:\(([^\n()]*)\))?') # Regex of macro names to exclude. - _excluded_name_re = re.compile(r'_(?:GET|IS|OF)_|_(?:BASE|FLAG|MASK)\Z') + _excluded_name_re = re.compile(r'_(?:GET|HAS|IS|OF)_|_(?:BASE|FLAG|MASK)\Z') # Additional excluded macros. _excluded_names = set([ # Macros that provide an alternative way to build the same From 1295140bcaedb09d425f74643c3057e5daafb6e2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 17:46:42 +0200 Subject: [PATCH 10/14] Read patch files instead of taking a single patch as input Having separate patch files has several benefits: * They're available for integrators who wouldn't use our script to test compliance. * We keep them separate so they're easier for us to keep track of, and apply separately if needed. * No need to cheat with unchanged empty lines (normally represented by a line containing a single space in a patch file) to keep `check_files.py` happy. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 28 ++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index c7bdd37e3..7deca5ecf 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -10,8 +10,10 @@ # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later import argparse +import glob import os import re +import shlex import shutil import subprocess import sys @@ -25,7 +27,7 @@ #pylint: disable=too-many-branches,too-many-statements,too-many-locals def test_compliance(library_build_dir: str, psa_arch_tests_ref: str, - patch: str, + patch_files: List[str], expected_failures: List[int]) -> int: """Check out and run compliance tests. @@ -59,12 +61,12 @@ def test_compliance(library_build_dir: str, subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, psa_arch_tests_ref]) subprocess.check_call(['git', 'checkout', '--force', 'FETCH_HEAD']) - if patch: + if patch_files: subprocess.check_call(['git', 'reset', '--hard']) - subprocess.run(['patch', '-p1'], - check=True, - encoding='utf-8', - input=patch) + for patch_file in patch_files: + abs_path = os.path.abspath(os.path.join(root_dir, patch_file)) + subprocess.check_call(['patch -p1 <' + shlex.quote(abs_path)], + shell=True) build_dir = 'api-tests/build' try: @@ -143,15 +145,15 @@ def test_compliance(library_build_dir: str, os.chdir(root_dir) def main(psa_arch_tests_ref: str, - patch: str = '', expected_failures: Optional[List[int]] = None) -> None: """Command line entry point. psa_arch_tests_ref: tag or sha to use for the arch-tests. - patch: patch to apply to the arch-tests with ``patch -p1``. expected_failures: default list of expected failures. """ build_dir = 'out_of_source_build' + default_patch_directory = os.path.join(build_tree.guess_project_root(), + 'scripts/data_files/psa-arch-tests') # pylint: disable=invalid-name parser = argparse.ArgumentParser() @@ -161,6 +163,9 @@ def main(psa_arch_tests_ref: str, help='''set the list of test codes which are expected to fail from the command line. If omitted the list given by EXPECTED_FAILURES (inside the script) is used.''') + parser.add_argument('--patch-directory', nargs=1, + default=default_patch_directory, + help='Directory containing patches (*.patch) to apply to psa-arch-tests') args = parser.parse_args() if args.build_dir is not None: @@ -173,7 +178,12 @@ def main(psa_arch_tests_ref: str, else: expected_failures_list = expected_failures + if args.patch_directory: + patch_files = glob.glob(os.path.join(args.patch_directory, '*.patch')) + else: + patch_files = [] + sys.exit(test_compliance(build_dir, psa_arch_tests_ref, - patch, + patch_files, expected_failures_list)) From 088a99ee205f033c72d392c1e0781bb2c7885bd0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 18:36:18 +0200 Subject: [PATCH 11/14] Allow framework and consuming branch to have Python scripts with the same name mypy can't deal with two modules with the same basename on its command line. We don't normally want modules with the same name in different directories, to avoid confusion, but it can happen occasionally while moving files across repositories. Signed-off-by: Gilles Peskine --- scripts/check-python-files.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/check-python-files.sh b/scripts/check-python-files.sh index a51766f71..171f8a961 100755 --- a/scripts/check-python-files.sh +++ b/scripts/check-python-files.sh @@ -62,7 +62,14 @@ $PYTHON -m pylint framework/scripts/*.py framework/scripts/mbedtls_framework/*.p echo echo 'Running mypy ...' -$PYTHON -m mypy framework/scripts/*.py framework/scripts/mbedtls_framework/*.py scripts/*.py tests/scripts/*.py || - ret=1 +$PYTHON -m mypy framework/scripts/*.py framework/scripts/mbedtls_framework/*.py || { + echo >&2 "mypy reported errors in the framework" + ret=1 +} + +$PYTHON -m mypy scripts/*.py tests/scripts/*.py || { + echo >&2 "pylint reported errors in the parent repository" + ret=1 +} exit $ret From 581da21d7b240ec037be1da1849e54f1681ca70c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 21:59:31 +0200 Subject: [PATCH 12/14] Allow patch files to have trailing whitespace An unchanged blank line results in a line containing a single space in the diff. Signed-off-by: Gilles Peskine --- scripts/check_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_files.py b/scripts/check_files.py index d3a61c1d6..3db3dcb2f 100755 --- a/scripts/check_files.py +++ b/scripts/check_files.py @@ -310,7 +310,7 @@ class TrailingWhitespaceIssueTracker(LineIssueTracker): """Track lines with trailing whitespace.""" heading = "Trailing whitespace:" - suffix_exemptions = frozenset([".dsp", ".md"]) + suffix_exemptions = frozenset([".diff", ".dsp", ".md", ".patch"]) def issue_with_line(self, line, _filepath, _line_number): return line.rstrip(b"\r\n") != line.rstrip() From aebb1faf3b2c119ff3ccd1b9c657dc91090ba808 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 22:01:38 +0200 Subject: [PATCH 13/14] Sort patch files for reproducibility Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 7deca5ecf..9ffdadffa 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -179,7 +179,8 @@ def main(psa_arch_tests_ref: str, expected_failures_list = expected_failures if args.patch_directory: - patch_files = glob.glob(os.path.join(args.patch_directory, '*.patch')) + patch_file_glob = os.path.join(args.patch_directory, '*.patch') + patch_files = sorted(glob.glob(patch_file_glob)) else: patch_files = [] From 44ea7135d68f63c493e7be0bed5f5d609b721a86 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Sep 2025 22:04:07 +0200 Subject: [PATCH 14/14] Simplify passing a file to subprocess stdin Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_compliance.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/mbedtls_framework/psa_compliance.py b/scripts/mbedtls_framework/psa_compliance.py index 9ffdadffa..aed3a1a1e 100644 --- a/scripts/mbedtls_framework/psa_compliance.py +++ b/scripts/mbedtls_framework/psa_compliance.py @@ -13,7 +13,6 @@ import glob import os import re -import shlex import shutil import subprocess import sys @@ -64,9 +63,9 @@ def test_compliance(library_build_dir: str, if patch_files: subprocess.check_call(['git', 'reset', '--hard']) for patch_file in patch_files: - abs_path = os.path.abspath(os.path.join(root_dir, patch_file)) - subprocess.check_call(['patch -p1 <' + shlex.quote(abs_path)], - shell=True) + with open(os.path.join(root_dir, patch_file), 'rb') as patch: + subprocess.check_call(['patch', '-p1'], + stdin=patch) build_dir = 'api-tests/build' try: