Merge pull request #5094 from bensze01/test_psa_compliance

Run the PSA Compliance test suite in all.sh
This commit is contained in:
Manuel Pégourié-Gonnard 2021-11-17 14:09:57 +01:00 committed by GitHub
commit 9b9fbda912
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 179 additions and 0 deletions

3
.gitignore vendored
View file

@ -41,6 +41,9 @@ massif-*
# Generated documentation:
/apidoc
# PSA Crypto compliance test repo, cloned by test_psa_compliance.py
/psa-arch-tests
# Editor navigation files:
/GPATH
/GRTAGS

View file

@ -2769,6 +2769,26 @@ component_test_zeroize () {
unset gdb_disable_aslr
}
component_test_psa_compliance () {
msg "build: make, default config (out-of-box), libmbedcrypto.a only"
make -C library libmbedcrypto.a
msg "unit test: test_psa_compliance.py"
./tests/scripts/test_psa_compliance.py
}
support_test_psa_compliance () {
# psa-compliance-tests only supports CMake >= 3.10.0
ver="$(cmake --version)"
ver="${ver#cmake version }"
ver_major="${ver%%.*}"
ver="${ver#*.}"
ver_minor="${ver%%.*}"
[ "$ver_major" -eq 3 ] && [ "$ver_minor" -ge 10 ]
}
component_check_python_files () {
msg "Lint: Python scripts"
tests/scripts/check-python-files.sh

View file

@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Run the PSA Cryto API compliance test suite.
Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
then complie and run the test suite. The clone is stored at <Mbed TLS root>/psa-arch-tests.
Known defects in either the test suite or mbedtls - identified by their test number - are ignored,
while unexpected failures AND successes are reported as errors,
to help keep the list of known defects as up to date as possible.
"""
# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import shutil
import subprocess
import sys
# PSA Compliance tests we expect to fail due to known defects in Mbed TLS (or the test suite)
# The test numbers correspond to the numbers used by the console output of the test suite.
# Test number 2xx corresponds to the files in the folder
# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
EXPECTED_FAILURES = {
# psa_key_derivation_output_key() returns PSA_ERROR_NOT_PERMITTED instead of
# PSA_ERROR_BAD_STATE when called after the operation was aborted.
# - Tracked in issue #5143
221,
# psa_aead_[encrypt/decrypt]() returns PSA_ERROR_NOT_SUPPORTED instead of
# PSA_ERROR_INVALID_ARGUMENT when called with an invalid nonce.
# - Tracked in issue #5144
224, 225,
# Multipart CCM is not supported.
# - Tracked in issue #3721
252, 253, 254, 255, 256, 257, 258, 259, 261,
# psa_hash_suspend() and psa_hash_resume() are not supported.
# - Tracked in issue #3274
262, 263
}
# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches
# that allow it to build with MbedTLS 3, and fixes a couple of issues in the compliance test suite.
# These fixes allow the tests numbered 216, 248 and 249 to complete successfully.
#
# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag.
# - Tracked in issue #5145
#
# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3
PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git'
PSA_ARCH_TESTS_REF = 'fixes-for-mbedtls-3'
#pylint: disable=too-many-branches,too-many-statements
def main():
mbedtls_dir = os.getcwd()
if not os.path.exists('library/libmbedcrypto.a'):
subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a'])
psa_arch_tests_dir = 'psa-arch-tests'
os.makedirs(psa_arch_tests_dir, exist_ok=True)
try:
os.chdir(psa_arch_tests_dir)
# Reuse existing local clone
subprocess.check_call(['git', 'init'])
subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
build_dir = 'api-tests/build'
try:
shutil.rmtree(build_dir)
except FileNotFoundError:
pass
os.mkdir(build_dir)
os.chdir(build_dir)
#pylint: disable=bad-continuation
subprocess.check_call([
'cmake', '..',
'-GUnix Makefiles',
'-DTARGET=tgt_dev_apis_stdc',
'-DTOOLCHAIN=HOST_GCC',
'-DSUITE=CRYPTO',
'-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir),
'-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir)
])
subprocess.check_call(['cmake', '--build', '.'])
proc = subprocess.Popen(['./psa-arch-tests-crypto'],
bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
test_re = re.compile(
'^TEST: (?P<test_num>[0-9]*)|'
'^TEST RESULT: (?P<test_result>FAILED|PASSED)'
)
test = -1
unexpected_successes = set(EXPECTED_FAILURES)
expected_failures = []
unexpected_failures = []
for line in proc.stdout:
print(line, end='')
match = test_re.match(line)
if match is not None:
groupdict = match.groupdict()
test_num = groupdict['test_num']
if test_num is not None:
test = int(test_num)
elif groupdict['test_result'] == 'FAILED':
try:
unexpected_successes.remove(test)
expected_failures.append(test)
print('Expected failure, ignoring')
except KeyError:
unexpected_failures.append(test)
print('ERROR: Unexpected failure')
elif test in unexpected_successes:
print('ERROR: Unexpected success')
proc.wait()
print()
print('***** test_psa_compliance.py report ******')
print()
print('Expected failures:', ', '.join(str(i) for i in expected_failures))
print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
print()
if unexpected_successes or unexpected_failures:
if unexpected_successes:
print('Unexpected successes encountered.')
print('Please remove the corresponding tests from '
'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
print()
print('FAILED')
return 1
else:
print('SUCCESS')
return 0
finally:
os.chdir(mbedtls_dir)
if __name__ == '__main__':
sys.exit(main())