yuzu-emu/mbedtls
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #6435 from mpg/improve-outcome-analysis

Browse source 
Improve outcome analysis for driver-only builds
This commit is contained in:
Gilles Peskine 2022-10-19 15:54:17 +02:00 committed by GitHub
commit 149c1516e7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 83 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
docs/architecture/psa-migration/outcome-analysis.sh
View file

@ -31,13 +31,25 @@ DRIVER_COMPONENT=test_psa_crypto_config_accel_hash_use_psa
# A similar configuration to that of the component, except without drivers,
# for comparison.
reference_config () {
scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
scripts/config.py unset MBEDTLS_PKCS1_V21
scripts/config.py unset MBEDTLS_X509_RSASSA_PSS_SUPPORT
# start with full
scripts/config.py full
# use PSA config and disable driver-less algs as in the component
scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER
scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING
# disable options as in the component
# (no need to disable whole modules, we'll just skip their test suite)
scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
}
# Space-separated list of test suites of interest.
SUITES="rsa pkcs1_v15 pk pkparse pkwrite"
# Space-separated list of test suites to ignore:
# if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored.
IGNORE="md mdx shax" # accelerated
IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG)
IGNORE="$IGNORE psa_crypto_init" # needs internal RNG
IGNORE="$IGNORE hkdf" # disabled in the all.sh component tested
# Compare only "reference vs driver" or also "before vs after"?
BEFORE_AFTER=1 # 0 or 1
# ----- END edit this -----
set -eu
@ -53,26 +65,28 @@ record() {
make check
}
# save current HEAD
HEAD=$(git branch --show-current)
if [ "$BEFORE_AFTER" -eq 1 ]; then
# save current HEAD
HEAD=$(git branch --show-current)
# get the numbers before this PR for default and full
cleanup
git checkout $(git merge-base HEAD development)
record "before-default"
# get the numbers before this PR for default and full
cleanup
git checkout $(git merge-base HEAD development)
record "before-default"
cleanup
scripts/config.py full
record "before-full"
cleanup
scripts/config.py full
record "before-full"
# get the numbers now for default and full
cleanup
git checkout $HEAD
record "after-default"
# get the numbers now for default and full
cleanup
git checkout $HEAD
record "after-default"
cleanup
scripts/config.py full
record "after-full"
cleanup
scripts/config.py full
record "after-full"
fi
# get the numbers now for driver-only and reference
cleanup
@ -81,10 +95,27 @@ record "reference"
cleanup
export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-drivers.csv"
export SKIP_SSL_OPT_COMPAT_SH=1
tests/scripts/all.sh -k test_psa_crypto_config_accel_hash_use_psa
# analysis
populate_suites () {
SUITES=''
make generated_files >/dev/null
data_files=$(cd tests/suites && echo *.data)
for data in $data_files; do
suite=${data#test_suite_}
suite=${suite%.data}
suite_base=${suite%%.*}
case " $IGNORE " in
*" $suite_base "*) :;;
*) SUITES="$SUITES $suite";;
esac
done
make neat
}
compare_suite () {
ref="outcome-$1.csv"
new="outcome-$2.csv"
@ -98,19 +129,35 @@ compare_suite () {
nb_ref=$(wc -l <skipped-ref)
nb_new=$(wc -l <skipped-new)
printf "%12s: total %3d; skipped %3d -> %3d\n" \
printf "%36s: total %4d; skipped %4d -> %4d\n" \
$suite $total $nb_ref $nb_new
diff skipped-ref skipped-new | grep '^> ' || true
if diff skipped-ref skipped-new | grep '^> '; then
ret=1
else
ret=0
fi
rm skipped-ref skipped-new
return $ret
}
compare_builds () {
printf "\n*** Comparing $1 -> $2 ***\n"
failed=''
for suite in $SUITES; do
compare_suite "$1" "$2" "$suite"
if compare_suite "$1" "$2" "$suite"; then :; else
failed="$failed $suite"
fi
done
if [ -z "$failed" ]; then
printf "No coverage gap found.\n"
else
printf "Suites with less coverage:%s\n" "$failed"
fi
}
compare_builds before-default after-default
compare_builds before-full after-full
populate_suites
if [ "$BEFORE_AFTER" -eq 1 ]; then
compare_builds before-default after-default
compare_builds before-full after-full
fi
compare_builds reference drivers

15
tests/scripts/all.sh
View file

@ -2043,7 +2043,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () {
# Also unset MD_C and things that depend on it;
# see component_test_crypto_full_no_md.
scripts/config.py unset MBEDTLS_MD_C
scripts/config.py unset MBEDTLS_HKDF_C
scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation
scripts/config.py unset MBEDTLS_HMAC_DRBG_C
scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
@ -2063,11 +2063,16 @@ component_test_psa_crypto_config_accel_hash_use_psa () {
msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
make test
msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
tests/ssl-opt.sh
# hidden option: when running outcome-analysis.sh, we can skip this
if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then
msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
tests/ssl-opt.sh
msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
tests/compat.sh
msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
tests/compat.sh
else
echo "skip ssl-opt.sh and compat.sh"
fi
}
component_test_psa_crypto_config_accel_cipher () {