From 80b31c56eba2634ddc60d3cf8e2cdc7eb9c639ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 19 Oct 2021 15:05:36 +0200 Subject: [PATCH 01/15] Run the PSA Compliance test suite in all.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a component to all.sh which clones, builds and runs the compliance test suite. Signed-off-by: Bence Szépkúti --- tests/scripts/all.sh | 13 ++++ tests/scripts/test_psa_compliance.py | 96 ++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100755 tests/scripts/test_psa_compliance.py diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 099174372..68163559a 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2756,6 +2756,19 @@ component_test_zeroize () { unset gdb_disable_aslr } +component_test_psa_compliance () { + msg "build: make, default config (out-of-box), libmbedcrypto.a only" + make library/libmbedcrypto.a + + msg "unit test: test_psa_compliance.py" + ./tests/scripts/test_psa_compliance.py +} + +support_test_psa_compliance () { + local ver=($(cmake --version | sed 's/cmake version //; y/./ /; q')) + [ "${ver[0]}" -eq 3 ] && [ "${ver[1]}" -ge 10 ] +} + component_check_python_files () { msg "Lint: Python scripts" tests/scripts/check-python-files.sh diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py new file mode 100755 index 000000000..07fa76e60 --- /dev/null +++ b/tests/scripts/test_psa_compliance.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +#pylint: disable=missing-module-docstring +import os +import re +import shutil +import subprocess +import sys + +EXPECTED_FAILURES = { + 216, 221, 224, 225, 248, 249, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263 +} +PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' +PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' + +#pylint: disable=too-many-statements +def main(): + mbedtls_dir = os.getcwd() + + mbedcrypto_lib = 'library/libmbedcrypto.a' + if not os.path.exists(mbedcrypto_lib): + subprocess.check_call(['make', mbedcrypto_lib]) + + psa_arch_tests_dir = 'psa-arch-tests' + try: + os.mkdir(psa_arch_tests_dir) + except FileExistsError: + pass + os.chdir(psa_arch_tests_dir) + + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) + + build_dir = 'api-tests/build' + try: + shutil.rmtree(build_dir) + except FileNotFoundError: + pass + os.mkdir(build_dir) + os.chdir(build_dir) + + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '..', '-GUnix Makefiles', + '-DTARGET=tgt_dev_apis_stdc', + '-DTOOLCHAIN=HOST_GCC', + '-DSUITE=CRYPTO', + '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), + '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) + ]) + subprocess.check_call(['cmake', '--build', '.']) + + proc = subprocess.Popen(['./psa-arch-tests-crypto'], + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + + test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: FAILED)') + test = -1 + unexpected_successes = set(EXPECTED_FAILURES) + expected_failures = [] + unexpected_failures = [] + for line in proc.stdout: + print(line[:-1]) + match = test_re.match(line) + if match is not None: + if match.group(1) is not None: + test = int(match.group(1)) + else: + try: + unexpected_successes.remove(test) + expected_failures.append(test) + except KeyError: + unexpected_failures.append(test) + proc.wait() + + print() + print('***** test_psa_compliance.py report ******') + print() + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) + print() + if unexpected_successes or unexpected_failures: + if unexpected_successes: + print('Unexpected successes encountered.') + #pylint: disable=line-too-long + print('Please remove the corresponding tests from EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print() + print('FAILED') + sys.exit(1) + else: + os.chdir(mbedtls_dir) + shutil.rmtree(psa_arch_tests_dir) + print('SUCCESS') + +if __name__ == '__main__': + main() From ca9236b0c56ac290e9f2c8da15330914dbd8bdaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 25 Oct 2021 19:29:07 +0200 Subject: [PATCH 02/15] Make the changes easier to backport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code replaced in this patch was not compatible with the development_2.x branch. Signed-off-by: Bence Szépkúti --- tests/scripts/all.sh | 12 +++++++++--- tests/scripts/test_psa_compliance.py | 5 ++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 68163559a..d86a9f773 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2758,15 +2758,21 @@ component_test_zeroize () { component_test_psa_compliance () { msg "build: make, default config (out-of-box), libmbedcrypto.a only" - make library/libmbedcrypto.a + make -C library libmbedcrypto.a msg "unit test: test_psa_compliance.py" ./tests/scripts/test_psa_compliance.py } support_test_psa_compliance () { - local ver=($(cmake --version | sed 's/cmake version //; y/./ /; q')) - [ "${ver[0]}" -eq 3 ] && [ "${ver[1]}" -ge 10 ] + ver="$(cmake --version)" + ver="${ver#cmake version }" + ver_major="${ver%%.*}" + + ver="${ver#*.}" + ver_minor="${ver%%.*}" + + [ "$ver_major" -eq 3 ] && [ "$ver_minor" -ge 10 ] } component_check_python_files () { diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 07fa76e60..d6fe8c440 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -16,9 +16,8 @@ PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' def main(): mbedtls_dir = os.getcwd() - mbedcrypto_lib = 'library/libmbedcrypto.a' - if not os.path.exists(mbedcrypto_lib): - subprocess.check_call(['make', mbedcrypto_lib]) + if not os.path.exists('library/libmbedcrypto.a'): + subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a']) psa_arch_tests_dir = 'psa-arch-tests' try: From d2ea2c0df3399a55d96274d34b85aa41a775bf10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 25 Oct 2021 20:58:14 +0200 Subject: [PATCH 03/15] Indicate errors interleaved with test suite output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indicate whether a success or failure is unexpected, or expected and ignored as they happen. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index d6fe8c440..aa0a480e5 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -52,7 +52,7 @@ def main(): proc = subprocess.Popen(['./psa-arch-tests-crypto'], bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) - test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: FAILED)') + test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: (FAILED|PASSED))') test = -1 unexpected_successes = set(EXPECTED_FAILURES) expected_failures = [] @@ -63,12 +63,16 @@ def main(): if match is not None: if match.group(1) is not None: test = int(match.group(1)) - else: + elif match.group(2) == 'FAILED': try: unexpected_successes.remove(test) expected_failures.append(test) + print('Expected failure, ignoring') except KeyError: unexpected_failures.append(test) + print('ERROR: Unexpected failure') + elif test in unexpected_successes: + print('ERROR: Unexpected success') proc.wait() print() From c2bac00530bb010015090fa9c38885ca1d41f1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 25 Oct 2021 20:58:14 +0200 Subject: [PATCH 04/15] Use print(end='') to silence double newline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index aa0a480e5..ca9387954 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -58,7 +58,7 @@ def main(): expected_failures = [] unexpected_failures = [] for line in proc.stdout: - print(line[:-1]) + print(line, end='') match = test_re.match(line) if match is not None: if match.group(1) is not None: From 83aa604ce5d6b3de956de2a116b8fa1c6caefc22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 29 Oct 2021 12:06:19 +0200 Subject: [PATCH 05/15] Simplify regex and use named capture groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index ca9387954..dfd23938a 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -52,7 +52,10 @@ def main(): proc = subprocess.Popen(['./psa-arch-tests-crypto'], bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) - test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: (FAILED|PASSED))') + test_re = re.compile( + '^TEST: (?P[0-9]*)|' + '^TEST RESULT: (?PFAILED|PASSED)' + ) test = -1 unexpected_successes = set(EXPECTED_FAILURES) expected_failures = [] @@ -61,9 +64,11 @@ def main(): print(line, end='') match = test_re.match(line) if match is not None: - if match.group(1) is not None: - test = int(match.group(1)) - elif match.group(2) == 'FAILED': + groupdict = match.groupdict() + test_num = groupdict['test_num'] + if test_num is not None: + test = int(test_num) + elif groupdict['test_result'] == 'FAILED': try: unexpected_successes.remove(test) expected_failures.append(test) From 449781fda774db94b3b0366002047817e774f683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 13:41:14 +0100 Subject: [PATCH 06/15] Fix pylint errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index dfd23938a..41003d80d 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 -#pylint: disable=missing-module-docstring +"""Run the PSA Cryto API compliance test suite. +Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, +then complie and run the test suite. +Known defects in either the test suite or mbedtls - identified by their test number - are ignored, +while unexpected failures AND successes are reported as errors, +to help keep the list of known defects as up to date as possible. +""" import os import re import shutil @@ -90,8 +96,8 @@ def main(): if unexpected_successes or unexpected_failures: if unexpected_successes: print('Unexpected successes encountered.') - #pylint: disable=line-too-long - print('Please remove the corresponding tests from EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print('Please remove the corresponding tests from ' + 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') print() print('FAILED') sys.exit(1) From 34b5f5634407a7b9ce5c3a36fb853276caed2985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 13:48:39 +0100 Subject: [PATCH 07/15] Make main() suitable to being called from python MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't call sys.exit(), and don't clobber the working directory. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 146 ++++++++++++++------------- 1 file changed, 75 insertions(+), 71 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 41003d80d..7d7192f06 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -18,7 +18,7 @@ EXPECTED_FAILURES = { PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' -#pylint: disable=too-many-statements +#pylint: disable=too-many-branches,too-many-statements def main(): mbedtls_dir = os.getcwd() @@ -30,81 +30,85 @@ def main(): os.mkdir(psa_arch_tests_dir) except FileExistsError: pass - os.chdir(psa_arch_tests_dir) - - subprocess.check_call(['git', 'init']) - subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) - subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) - - build_dir = 'api-tests/build' try: - shutil.rmtree(build_dir) - except FileNotFoundError: - pass - os.mkdir(build_dir) - os.chdir(build_dir) + os.chdir(psa_arch_tests_dir) - #pylint: disable=bad-continuation - subprocess.check_call([ - 'cmake', '..', '-GUnix Makefiles', - '-DTARGET=tgt_dev_apis_stdc', - '-DTOOLCHAIN=HOST_GCC', - '-DSUITE=CRYPTO', - '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), - '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) - ]) - subprocess.check_call(['cmake', '--build', '.']) + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) - proc = subprocess.Popen(['./psa-arch-tests-crypto'], - bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + build_dir = 'api-tests/build' + try: + shutil.rmtree(build_dir) + except FileNotFoundError: + pass + os.mkdir(build_dir) + os.chdir(build_dir) - test_re = re.compile( - '^TEST: (?P[0-9]*)|' - '^TEST RESULT: (?PFAILED|PASSED)' - ) - test = -1 - unexpected_successes = set(EXPECTED_FAILURES) - expected_failures = [] - unexpected_failures = [] - for line in proc.stdout: - print(line, end='') - match = test_re.match(line) - if match is not None: - groupdict = match.groupdict() - test_num = groupdict['test_num'] - if test_num is not None: - test = int(test_num) - elif groupdict['test_result'] == 'FAILED': - try: - unexpected_successes.remove(test) - expected_failures.append(test) - print('Expected failure, ignoring') - except KeyError: - unexpected_failures.append(test) - print('ERROR: Unexpected failure') - elif test in unexpected_successes: - print('ERROR: Unexpected success') - proc.wait() + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '..', + '-GUnix Makefiles', + '-DTARGET=tgt_dev_apis_stdc', + '-DTOOLCHAIN=HOST_GCC', + '-DSUITE=CRYPTO', + '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), + '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) + ]) + subprocess.check_call(['cmake', '--build', '.']) - print() - print('***** test_psa_compliance.py report ******') - print() - print('Expected failures:', ', '.join(str(i) for i in expected_failures)) - print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) - print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) - print() - if unexpected_successes or unexpected_failures: - if unexpected_successes: - print('Unexpected successes encountered.') - print('Please remove the corresponding tests from ' - 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') - print() - print('FAILED') - sys.exit(1) - else: + proc = subprocess.Popen(['./psa-arch-tests-crypto'], + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + + test_re = re.compile( + '^TEST: (?P[0-9]*)|' + '^TEST RESULT: (?PFAILED|PASSED)' + ) + test = -1 + unexpected_successes = set(EXPECTED_FAILURES) + expected_failures = [] + unexpected_failures = [] + for line in proc.stdout: + print(line, end='') + match = test_re.match(line) + if match is not None: + groupdict = match.groupdict() + test_num = groupdict['test_num'] + if test_num is not None: + test = int(test_num) + elif groupdict['test_result'] == 'FAILED': + try: + unexpected_successes.remove(test) + expected_failures.append(test) + print('Expected failure, ignoring') + except KeyError: + unexpected_failures.append(test) + print('ERROR: Unexpected failure') + elif test in unexpected_successes: + print('ERROR: Unexpected success') + proc.wait() + + print() + print('***** test_psa_compliance.py report ******') + print() + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) + print() + if unexpected_successes or unexpected_failures: + if unexpected_successes: + print('Unexpected successes encountered.') + print('Please remove the corresponding tests from ' + 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print() + print('FAILED') + return 1 + else: + shutil.rmtree(psa_arch_tests_dir) + print('SUCCESS') + return 0 + finally: os.chdir(mbedtls_dir) - shutil.rmtree(psa_arch_tests_dir) - print('SUCCESS') if __name__ == '__main__': - main() + sys.exit(main()) From 67fb3149c01dd5e121980f1de8061124288eb8d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 14:01:08 +0100 Subject: [PATCH 08/15] Add licence header to script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 7d7192f06..d94f6c242 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -6,6 +6,22 @@ Known defects in either the test suite or mbedtls - identified by their test num while unexpected failures AND successes are reported as errors, to help keep the list of known defects as up to date as possible. """ + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import re import shutil From c63d1605ab69f8aff97cf9c8678bbfdfd6f8fe00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 14:06:40 +0100 Subject: [PATCH 09/15] Make directory creation code more compact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index d94f6c242..33207c014 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -42,10 +42,7 @@ def main(): subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a']) psa_arch_tests_dir = 'psa-arch-tests' - try: - os.mkdir(psa_arch_tests_dir) - except FileExistsError: - pass + os.makedirs(psa_arch_tests_dir, exist_ok=True) try: os.chdir(psa_arch_tests_dir) From b3818412bcb043503bfe287de0e1135502a33448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 3 Nov 2021 11:32:51 +0100 Subject: [PATCH 10/15] Keep local clone around even if the test succeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- .gitignore | 3 +++ tests/scripts/test_psa_compliance.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0792920a8..e86092c45 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,9 @@ massif-* # Generated documentation: /apidoc +# PSA Crypto compliance test repo, cloned by test_psa_complaince.py +/psa-arch-tests + # Editor navigation files: /GPATH /GRTAGS diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 33207c014..2f67f08c8 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Run the PSA Cryto API compliance test suite. Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, -then complie and run the test suite. +then complie and run the test suite. The clone is stored at /psa-arch-tests. Known defects in either the test suite or mbedtls - identified by their test number - are ignored, while unexpected failures AND successes are reported as errors, to help keep the list of known defects as up to date as possible. @@ -46,6 +46,7 @@ def main(): try: os.chdir(psa_arch_tests_dir) + # Reuse existing local clone subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) @@ -117,7 +118,6 @@ def main(): print('FAILED') return 1 else: - shutil.rmtree(psa_arch_tests_dir) print('SUCCESS') return 0 finally: From ef0d02ed317c431a9eaa3ec8c3ba58cf8952d673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 3 Nov 2021 11:36:09 +0100 Subject: [PATCH 11/15] Explain why support_test_psa_compliance is needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index d86a9f773..449b213ff 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2765,6 +2765,7 @@ component_test_psa_compliance () { } support_test_psa_compliance () { + # psa-compliance-tests only supports CMake >= 3.10.0 ver="$(cmake --version)" ver="${ver#cmake version }" ver_major="${ver%%.*}" From 9e9aa5d2ebd4ffbeb6ac46d070dc389665e71e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Thu, 4 Nov 2021 16:39:48 +0100 Subject: [PATCH 12/15] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e86092c45..26986d60c 100644 --- a/.gitignore +++ b/.gitignore @@ -41,7 +41,7 @@ massif-* # Generated documentation: /apidoc -# PSA Crypto compliance test repo, cloned by test_psa_complaince.py +# PSA Crypto compliance test repo, cloned by test_psa_compliance.py /psa-arch-tests # Editor navigation files: From e2855c32b5d121efe4863782b6694761a600a848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 9 Nov 2021 17:33:57 +0100 Subject: [PATCH 13/15] Move to an updated fork of psa-arch-tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new fork was rebased on top of the upstream master, removing the need for most of the downstream patches we carried. On the other hand, the new fork includes a couple of fixes to problems that were not addressed by the original fork, or were introduced with the new version of psa-arch-tests. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 2f67f08c8..58cb8f1a9 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -29,10 +29,18 @@ import subprocess import sys EXPECTED_FAILURES = { - 216, 221, 224, 225, 248, 249, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263 + 221, 224, 225, 252, 253, 254, 255, 256, 257, 258, 259, 261, 262, 263 } -PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' -PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' + +# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches +# that allow it to build with MbedTLS 3, and fixes a couple of issues in the compliance test suite. +# These fixes allow the tests numbered 216, 248 and 249 to complete successfully. +# +# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag. +# +# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3 +PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git' +PSA_ARCH_TESTS_REF = 'fixes-for-mbedtls-3' #pylint: disable=too-many-branches,too-many-statements def main(): From cb288713264175dfab7987b67f1578b40803ca76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 9 Nov 2021 21:30:43 +0100 Subject: [PATCH 14/15] Document the values in EXPECTED_FAILURES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Including the issues where the corresponding defects are tracked. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 58cb8f1a9..31e3fce77 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -28,8 +28,28 @@ import shutil import subprocess import sys +# PSA Compliance tests we expect to fail due to known defects in Mbed TLS (or the test suite) +# The test numbers correspond to the numbers used by the console output of the test suite. +# Test number 2xx corresponds to the files in the folder +# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx EXPECTED_FAILURES = { - 221, 224, 225, 252, 253, 254, 255, 256, 257, 258, 259, 261, 262, 263 + # psa_key_derivation_output_key() returns PSA_ERROR_NOT_PERMITTED instead of + # PSA_ERROR_BAD_STATE when called after the operation was aborted. + # - Tracked in issue #5143 + 221, + + # psa_aead_[encrypt/decrypt]() returns PSA_ERROR_NOT_SUPPORTED instead of + # PSA_ERROR_INVALID_ARGUMENT when called with an invalid nonce. + # - Tracked in issue #5144 + 224, 225, + + # Multipart CCM is not supported. + # - Tracked in issue #3721 + 252, 253, 254, 255, 256, 257, 258, 259, 261, + + # psa_hash_suspend() and psa_hash_resume() are not supported. + # - Tracked in issue #3274 + 262, 263 } # We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches From b376eac5aca9b82659bc46f723c458bed02607ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 9 Nov 2021 22:13:46 +0100 Subject: [PATCH 15/15] Track upstreaming task in an issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 31e3fce77..2f6358132 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -57,6 +57,7 @@ EXPECTED_FAILURES = { # These fixes allow the tests numbered 216, 248 and 249 to complete successfully. # # Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag. +# - Tracked in issue #5145 # # Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3 PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git'