From 2bb2f15772b93d5878fb6e239d3e248d697d4ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 12 Oct 2022 10:57:31 +0200 Subject: [PATCH 1/5] Improve driver-only outcome-analysis script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of having a list of test suites of interest, have a list of suites to ignore and look at all the others. In order for this to only yield interesting results, we need to tune the reference configuration a bit, in order to exclude STREAM and ECB to more closely match the driver-based configuration. Signed-off-by: Manuel Pégourié-Gonnard --- .../psa-migration/outcome-analysis.sh | 52 ++++++++++++++++--- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh index 67510c144..f3566b20a 100755 --- a/docs/architecture/psa-migration/outcome-analysis.sh +++ b/docs/architecture/psa-migration/outcome-analysis.sh @@ -31,13 +31,23 @@ DRIVER_COMPONENT=test_psa_crypto_config_accel_hash_use_psa # A similar configuration to that of the component, except without drivers, # for comparison. reference_config () { - scripts/config.py set MBEDTLS_USE_PSA_CRYPTO - scripts/config.py unset MBEDTLS_PKCS1_V21 - scripts/config.py unset MBEDTLS_X509_RSASSA_PSS_SUPPORT + # start with full + scripts/config.py full + # use PSA config and disable driver-less algs as in the component + scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING + # disable options as in the component + # (no need to disable whole modules, we'll just skip their test suite) scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC + scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA } -# Space-separated list of test suites of interest. -SUITES="rsa pkcs1_v15 pk pkparse pkwrite" +# Space-separated list of test suites to ignore: +# if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored. +IGNORE="md mdx shax" # accelerated +IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG) +IGNORE="$IGNORE psa_crypto_init" # needs internal RNG +IGNORE="$IGNORE hkdf" # disabled # ----- END edit this ----- set -eu @@ -85,6 +95,22 @@ tests/scripts/all.sh -k test_psa_crypto_config_accel_hash_use_psa # analysis +populate_suites () { + SUITES='' + make generated_files >/dev/null + data_files=$(cd tests/suites && echo *.data) + for data in $data_files; do + suite=${data#test_suite_} + suite=${suite%.data} + suite_base=${suite%%.*} + case " $IGNORE " in + *" $suite_base "*) :;; + *) SUITES="$SUITES $suite";; + esac + done + make neat +} + compare_suite () { ref="outcome-$1.csv" new="outcome-$2.csv" @@ -98,19 +124,29 @@ compare_suite () { nb_ref=$(wc -l %3d\n" \ + printf "%36s: total %4d; skipped %4d -> %4d\n" \ $suite $total $nb_ref $nb_new - diff skipped-ref skipped-new | grep '^> ' || true + if diff skipped-ref skipped-new | grep '^> '; then + ret=1 + else + ret=0 + fi rm skipped-ref skipped-new + return $ret } compare_builds () { printf "\n*** Comparing $1 -> $2 ***\n" + failed='' for suite in $SUITES; do - compare_suite "$1" "$2" "$suite" + if compare_suite "$1" "$2" "$suite"; then :; else + failed="$failed $suite" + fi done + printf "suites with less coverage: %s\n" "$failed" } +populate_suites compare_builds before-default after-default compare_builds before-full after-full compare_builds reference drivers From f6e6df9dbf10c6fbaaa1253132b935959f4b673f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Mon, 17 Oct 2022 12:24:37 +0200 Subject: [PATCH 2/5] Add option for before-after or just ref-drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- .../psa-migration/outcome-analysis.sh | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh index f3566b20a..f777b8fb9 100755 --- a/docs/architecture/psa-migration/outcome-analysis.sh +++ b/docs/architecture/psa-migration/outcome-analysis.sh @@ -48,6 +48,8 @@ IGNORE="md mdx shax" # accelerated IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG) IGNORE="$IGNORE psa_crypto_init" # needs internal RNG IGNORE="$IGNORE hkdf" # disabled +# Compare only "reference vs driver" or also "before vs after"? +BEFORE_AFTER=1 # 0 or 1 # ----- END edit this ----- set -eu @@ -63,26 +65,28 @@ record() { make check } -# save current HEAD -HEAD=$(git branch --show-current) +if [ "$BEFORE_AFTER" -eq 1 ]; then + # save current HEAD + HEAD=$(git branch --show-current) -# get the numbers before this PR for default and full -cleanup -git checkout $(git merge-base HEAD development) -record "before-default" + # get the numbers before this PR for default and full + cleanup + git checkout $(git merge-base HEAD development) + record "before-default" -cleanup -scripts/config.py full -record "before-full" + cleanup + scripts/config.py full + record "before-full" -# get the numbers now for default and full -cleanup -git checkout $HEAD -record "after-default" + # get the numbers now for default and full + cleanup + git checkout $HEAD + record "after-default" -cleanup -scripts/config.py full -record "after-full" + cleanup + scripts/config.py full + record "after-full" +fi # get the numbers now for driver-only and reference cleanup @@ -147,6 +151,8 @@ compare_builds () { } populate_suites -compare_builds before-default after-default -compare_builds before-full after-full +if [ "$BEFORE_AFTER" -eq 1 ]; then + compare_builds before-default after-default + compare_builds before-full after-full +fi compare_builds reference drivers From b51051f1c76b69b02d65b7228ed526c213541380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Tue, 18 Oct 2022 09:42:30 +0200 Subject: [PATCH 3/5] Cosmetic improvement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- docs/architecture/psa-migration/outcome-analysis.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh index f777b8fb9..8c6f5e65e 100755 --- a/docs/architecture/psa-migration/outcome-analysis.sh +++ b/docs/architecture/psa-migration/outcome-analysis.sh @@ -147,7 +147,11 @@ compare_builds () { failed="$failed $suite" fi done - printf "suites with less coverage: %s\n" "$failed" + if [ -z "$failed" ]; then + printf "No coverage gap found.\n" + else + printf "Suites with less coverage:%s\n" "$failed" + fi } populate_suites From d92fb01419755046c32f4caa3a19c8a6371aaec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Tue, 18 Oct 2022 10:14:31 +0200 Subject: [PATCH 4/5] Skip bits not needed in outcome-analysis.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- docs/architecture/psa-migration/outcome-analysis.sh | 1 + tests/scripts/all.sh | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh index 8c6f5e65e..2134c5bb1 100755 --- a/docs/architecture/psa-migration/outcome-analysis.sh +++ b/docs/architecture/psa-migration/outcome-analysis.sh @@ -95,6 +95,7 @@ record "reference" cleanup export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-drivers.csv" +export SKIP_SSL_OPT_COMPAT_SH=1 tests/scripts/all.sh -k test_psa_crypto_config_accel_hash_use_psa # analysis diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index f1b2f0e29..e794d088a 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2063,11 +2063,16 @@ component_test_psa_crypto_config_accel_hash_use_psa () { msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" make test - msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" - tests/ssl-opt.sh + # hidden option: when running outcome-analysis.sh, we can skip this + if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then + msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" + tests/ssl-opt.sh - msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" - tests/compat.sh + msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" + tests/compat.sh + else + echo "skip sh scripts" + fi } component_test_psa_crypto_config_accel_cipher () { From 0dc40773d652858c3baa4c735a8e6f4d86d7c6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 19 Oct 2022 12:12:21 +0200 Subject: [PATCH 5/5] Improve comments & messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- docs/architecture/psa-migration/outcome-analysis.sh | 2 +- tests/scripts/all.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh index 2134c5bb1..81ab69183 100755 --- a/docs/architecture/psa-migration/outcome-analysis.sh +++ b/docs/architecture/psa-migration/outcome-analysis.sh @@ -47,7 +47,7 @@ reference_config () { IGNORE="md mdx shax" # accelerated IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG) IGNORE="$IGNORE psa_crypto_init" # needs internal RNG -IGNORE="$IGNORE hkdf" # disabled +IGNORE="$IGNORE hkdf" # disabled in the all.sh component tested # Compare only "reference vs driver" or also "before vs after"? BEFORE_AFTER=1 # 0 or 1 # ----- END edit this ----- diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index e794d088a..7b93fce68 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2043,7 +2043,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () { # Also unset MD_C and things that depend on it; # see component_test_crypto_full_no_md. scripts/config.py unset MBEDTLS_MD_C - scripts/config.py unset MBEDTLS_HKDF_C + scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation scripts/config.py unset MBEDTLS_HMAC_DRBG_C scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA @@ -2071,7 +2071,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () { msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA" tests/compat.sh else - echo "skip sh scripts" + echo "skip ssl-opt.sh and compat.sh" fi }