Merge pull request #7136 from yanrayw/5692-record-compatsh-test-cases

Record the outcome of each test case in compat.sh

docs/architecture/testing/test-framework.md

@@ -30,17 +30,23 @@ Each test case has a description which succinctly describes for a human audience
 
 #### SSL test case descriptions
 
-Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`.
+Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_test`.
 
 The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check_test_cases.py`.
 
+### SSL cipher suite tests
+
+Each test case in `compat.sh` has a description which succinctly describes for a human audience what the test does. The test description is `$TITLE` defined in `run_client`.
+
+The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, failure cause in `compat.sh` is not classified as `ssl-opt.sh`, so the information of failed log files are followed as prompt.
+
 ## Running tests
 
 ### Outcome file
 
 #### Generating an outcome file
 
-Unit tests and `ssl-opt.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
+Unit tests, `ssl-opt.sh` and `compat.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
 
 If you run `all.sh --outcome-file test-outcome.csv`, this collects the outcome of all the test cases in `test-outcome.csv`.
 
@@ -52,7 +58,7 @@ The outcome file has 6 fields:
 
 * **Platform**: a description of the platform, e.g. `Linux-x86_64` or `Linux-x86_64-gcc7-msan`.
 * **Configuration**: a unique description of the configuration (`mbedtls_config.h`).
-* **Test suite**: `test_suite_xxx` or `ssl-opt`.
+* **Test suite**: `test_suite_xxx`, `ssl-opt` or `compat`.
 * **Test case**: the description of the test case.
 * **Result**: one of `PASS`, `SKIP` or `FAIL`.
 * **Cause**: more information explaining the result.

tests/compat.sh

@@ -30,6 +30,11 @@ set -u
 # where it may output seemingly unlimited length error logs.
 ulimit -f 20971520
 
+ORIGINAL_PWD=$PWD
+if ! cd "$(dirname "$0")"; then
+    exit 125
+fi
+
 # initialise counters
 TESTS=0
 FAILED=0
@@ -77,6 +82,17 @@ else
     PEER_GNUTLS=""
 fi
 
+guess_config_name() {
+    if git diff --quiet ../include/mbedtls/mbedtls_config.h 2>/dev/null; then
+        echo "default"
+    else
+        echo "unknown"
+    fi
+}
+: ${MBEDTLS_TEST_OUTCOME_FILE=}
+: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"}
+: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
+
 # default values for options
 # /!\ keep this synchronised with:
 # - basic-build-test.sh
@@ -110,6 +126,8 @@ print_usage() {
     printf "            \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
     printf "  -M|--memcheck\tCheck memory leaks and errors.\n"
     printf "  -v|--verbose\tSet verbose output.\n"
+    printf "     --outcome-file\tFile where test outcomes are written\n"
+    printf "                   \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
 }
 
 get_options() {
@@ -139,6 +157,9 @@ get_options() {
             -M|--memcheck)
                 MEMCHECK=1
                 ;;
+            --outcome-file)
+                shift; MBEDTLS_TEST_OUTCOME_FILE=$1
+                ;;
             -h|--help)
                 print_usage
                 exit 0
@@ -805,6 +826,39 @@ wait_client_done() {
     echo "EXIT: $EXIT" >> $CLI_OUT
 }
 
+# record_outcome <outcome> [<failure-reason>]
+record_outcome() {
+    echo "$1"
+    if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
+        # The test outcome file has the format (in single line):
+        # platform;configuration;
+        # test suite name;test case description;
+        # PASS/FAIL/SKIP;[failure cause]
+        printf '%s;%s;%s;%s;%s;%s\n'                                    \
+            "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION"      \
+            "compat" "$TITLE"                                           \
+            "$1" "${2-}"                                                \
+            >> "$MBEDTLS_TEST_OUTCOME_FILE"
+    fi
+}
+
+# display additional information if test case fails
+report_fail() {
+    FAIL_PROMPT="outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
+    record_outcome "FAIL" "$FAIL_PROMPT"
+    cp $SRV_OUT c-srv-${TESTS}.log
+    cp $CLI_OUT c-cli-${TESTS}.log
+    echo "  ! $FAIL_PROMPT"
+
+    if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
+        echo "  ! server output:"
+        cat c-srv-${TESTS}.log
+        echo "  ! ==================================================="
+        echo "  ! client output:"
+        cat c-cli-${TESTS}.log
+    fi
+}
+
 # run_client PROGRAM_NAME STANDARD_CIPHER_SUITE PROGRAM_CIPHER_SUITE
 run_client() {
     # announce what we're going to do
@@ -817,7 +871,7 @@ run_client() {
     # should we skip?
     if [ "X$SKIP_NEXT" = "XYES" ]; then
         SKIP_NEXT="NO"
-        echo "SKIP"
+        record_outcome "SKIP"
         SKIPPED=$(( $SKIPPED + 1 ))
         return
     fi
@@ -911,26 +965,14 @@ run_client() {
     # report and count result
     case $RESULT in
         "0")
-            echo PASS
+            record_outcome "PASS"
             ;;
         "1")
-            echo SKIP
+            record_outcome "SKIP"
             SKIPPED=$(( $SKIPPED + 1 ))
             ;;
         "2")
-            echo FAIL
+            report_fail
-            cp $SRV_OUT c-srv-${TESTS}.log
-            cp $CLI_OUT c-cli-${TESTS}.log
-            echo "  ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
-
-            if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
-                echo "  ! server output:"
-                cat c-srv-${TESTS}.log
-                echo "  ! ==================================================="
-                echo "  ! client output:"
-                cat c-cli-${TESTS}.log
-            fi
-
             FAILED=$(( $FAILED + 1 ))
             ;;
     esac
@@ -942,13 +984,16 @@ run_client() {
 # MAIN
 #
 
-if cd $( dirname $0 ); then :; else
-    echo "cd $( dirname $0 ) failed" >&2
-    exit 1
-fi
-
 get_options "$@"
 
+# Make the outcome file path relative to the original directory, not
+# to .../tests
+case "$MBEDTLS_TEST_OUTCOME_FILE" in
+    [!/]*)
+        MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
+        ;;
+esac
+
 # sanity checks, avoid an avalanche of errors
 if [ ! -x "$M_SRV" ]; then
     echo "Command '$M_SRV' is not an executable file" >&2

tests/ssl-opt.sh

@@ -177,6 +177,9 @@ get_options() {
             -p|--preserve-logs)
                 PRESERVE_LOGS=1
                 ;;
+            --outcome-file)
+                shift; MBEDTLS_TEST_OUTCOME_FILE=$1
+                ;;
             --port)
                 shift; SRV_PORT=$1
                 ;;
@@ -200,14 +203,6 @@ get_options() {
     done
 }
 
-# Make the outcome file path relative to the original directory, not
-# to .../tests
-case "$MBEDTLS_TEST_OUTCOME_FILE" in
-    [!/]*)
-        MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
-        ;;
-esac
-
 # Read boolean configuration options from mbedtls_config.h for easy and quick
 # testing. Skip non-boolean options (with something other than spaces
 # and a comment after "#define SYMBOL"). The variable contains a
@@ -1791,6 +1786,14 @@ cleanup() {
 
 get_options "$@"
 
+# Make the outcome file path relative to the original directory, not
+# to .../tests
+case "$MBEDTLS_TEST_OUTCOME_FILE" in
+    [!/]*)
+        MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
+        ;;
+esac
+
 populate_enabled_hash_algs
 
 # Optimize filters: if $FILTER and $EXCLUDE can be expressed as shell