Merge pull request #6435 from mpg/improve-outcome-analysis

Improve outcome analysis for driver-only builds

docs/architecture/psa-migration/outcome-analysis.sh

@@ -31,13 +31,25 @@ DRIVER_COMPONENT=test_psa_crypto_config_accel_hash_use_psa
 # A similar configuration to that of the component, except without drivers,
 # for comparison.
 reference_config () {
-    scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
+    # start with full
-    scripts/config.py unset MBEDTLS_PKCS1_V21
+    scripts/config.py full
-    scripts/config.py unset MBEDTLS_X509_RSASSA_PSS_SUPPORT
+    # use PSA config and disable driver-less algs as in the component
+    scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
+    scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER
+    scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING
+    # disable options as in the component
+    # (no need to disable whole modules, we'll just skip their test suite)
     scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
+    scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
 }
-# Space-separated list of test suites of interest.
+# Space-separated list of test suites to ignore:
-SUITES="rsa pkcs1_v15 pk pkparse pkwrite"
+# if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored.
+IGNORE="md mdx shax" # accelerated
+IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG)
+IGNORE="$IGNORE psa_crypto_init" # needs internal RNG
+IGNORE="$IGNORE hkdf" # disabled in the all.sh component tested
+# Compare only "reference vs driver" or also "before vs after"?
+BEFORE_AFTER=1 # 0 or 1
 # ----- END edit this -----
 
 set -eu
@@ -53,26 +65,28 @@ record() {
     make check
 }
 
-# save current HEAD
+if [ "$BEFORE_AFTER" -eq 1 ]; then
-HEAD=$(git branch --show-current)
+    # save current HEAD
+    HEAD=$(git branch --show-current)
 
-# get the numbers before this PR for default and full
+    # get the numbers before this PR for default and full
-cleanup
+    cleanup
-git checkout $(git merge-base HEAD development)
+    git checkout $(git merge-base HEAD development)
-record "before-default"
+    record "before-default"
 
-cleanup
+    cleanup
-scripts/config.py full
+    scripts/config.py full
-record "before-full"
+    record "before-full"
 
-# get the numbers now for default and full
+    # get the numbers now for default and full
-cleanup
+    cleanup
-git checkout $HEAD
+    git checkout $HEAD
-record "after-default"
+    record "after-default"
 
-cleanup
+    cleanup
-scripts/config.py full
+    scripts/config.py full
-record "after-full"
+    record "after-full"
+fi
 
 # get the numbers now for driver-only and reference
 cleanup
@@ -81,10 +95,27 @@ record "reference"
 
 cleanup
 export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-drivers.csv"
+export SKIP_SSL_OPT_COMPAT_SH=1
 tests/scripts/all.sh -k test_psa_crypto_config_accel_hash_use_psa
 
 # analysis
 
+populate_suites () {
+    SUITES=''
+    make generated_files >/dev/null
+    data_files=$(cd tests/suites && echo *.data)
+    for data in $data_files; do
+        suite=${data#test_suite_}
+        suite=${suite%.data}
+        suite_base=${suite%%.*}
+        case " $IGNORE " in
+            *" $suite_base "*) :;;
+            *) SUITES="$SUITES $suite";;
+        esac
+    done
+    make neat
+}
+
 compare_suite () {
     ref="outcome-$1.csv"
     new="outcome-$2.csv"
@@ -98,19 +129,35 @@ compare_suite () {
     nb_ref=$(wc -l <skipped-ref)
     nb_new=$(wc -l <skipped-new)
 
-    printf "%12s: total %3d; skipped %3d -> %3d\n" \
+    printf "%36s: total %4d; skipped %4d -> %4d\n" \
             $suite      $total       $nb_ref $nb_new
-    diff skipped-ref skipped-new | grep '^> ' || true
+    if diff skipped-ref skipped-new | grep '^> '; then
+        ret=1
+    else
+        ret=0
+    fi
     rm skipped-ref skipped-new
+    return $ret
 }
 
 compare_builds () {
     printf "\n*** Comparing $1 -> $2 ***\n"
+    failed=''
     for suite in $SUITES; do
-        compare_suite "$1" "$2" "$suite"
+        if compare_suite "$1" "$2" "$suite"; then :; else
+            failed="$failed $suite"
+        fi
     done
+    if [ -z "$failed" ]; then
+        printf "No coverage gap found.\n"
+    else
+        printf "Suites with less coverage:%s\n" "$failed"
+    fi
 }
 
-compare_builds before-default after-default
+populate_suites
-compare_builds before-full after-full
+if [ "$BEFORE_AFTER" -eq 1 ]; then
+    compare_builds before-default after-default
+    compare_builds before-full after-full
+fi
 compare_builds reference drivers

tests/scripts/all.sh

@@ -2043,7 +2043,7 @@ component_test_psa_crypto_config_accel_hash_use_psa () {
     # Also unset MD_C and things that depend on it;
     # see component_test_crypto_full_no_md.
     scripts/config.py unset MBEDTLS_MD_C
-    scripts/config.py unset MBEDTLS_HKDF_C
+    scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation
     scripts/config.py unset MBEDTLS_HMAC_DRBG_C
     scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
     scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
@@ -2063,11 +2063,16 @@ component_test_psa_crypto_config_accel_hash_use_psa () {
     msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
     make test
 
-    msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
+    # hidden option: when running outcome-analysis.sh, we can skip this
-    tests/ssl-opt.sh
+    if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then
+        msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
+        tests/ssl-opt.sh
 
-    msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
+        msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
-    tests/compat.sh
+        tests/compat.sh
+    else
+        echo "skip ssl-opt.sh and compat.sh"
+    fi
 }
 
 component_test_psa_crypto_config_accel_cipher () {