Merge pull request #7136 from yanrayw/5692-record-compatsh-test-cases
Record the outcome of each test case in compat.sh
This commit is contained in:
Dave Rodgman
GitHub
commit
e183ecef3d
3 changed files with 86 additions and 32 deletions
|
|
@ -30,17 +30,23 @@ Each test case has a description which succinctly describes for a human audience
|
|||
|
||||
#### SSL test case descriptions
|
||||
|
||||
Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`.
|
||||
Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_test`.
|
||||
|
||||
The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check_test_cases.py`.
|
||||
|
||||
### SSL cipher suite tests
|
||||
|
||||
Each test case in `compat.sh` has a description which succinctly describes for a human audience what the test does. The test description is `$TITLE` defined in `run_client`.
|
||||
|
||||
The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, failure cause in `compat.sh` is not classified as `ssl-opt.sh`, so the information of failed log files are followed as prompt.
|
||||
|
||||
## Running tests
|
||||
|
||||
### Outcome file
|
||||
|
||||
#### Generating an outcome file
|
||||
|
||||
Unit tests and `ssl-opt.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
|
||||
Unit tests, `ssl-opt.sh` and `compat.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
|
||||
|
||||
If you run `all.sh --outcome-file test-outcome.csv`, this collects the outcome of all the test cases in `test-outcome.csv`.
|
||||
|
||||
|
|
@ -52,7 +58,7 @@ The outcome file has 6 fields:
|
|||
|
||||
* **Platform**: a description of the platform, e.g. `Linux-x86_64` or `Linux-x86_64-gcc7-msan`.
|
||||
* **Configuration**: a unique description of the configuration (`mbedtls_config.h`).
|
||||
* **Test suite**: `test_suite_xxx` or `ssl-opt`.
|
||||
* **Test suite**: `test_suite_xxx`, `ssl-opt` or `compat`.
|
||||
* **Test case**: the description of the test case.
|
||||
* **Result**: one of `PASS`, `SKIP` or `FAIL`.
|
||||
* **Cause**: more information explaining the result.
|
||||
|
|
|
|||
|
|
@ -30,6 +30,11 @@ set -u
|
|||
# where it may output seemingly unlimited length error logs.
|
||||
ulimit -f 20971520
|
||||
|
||||
ORIGINAL_PWD=$PWD
|
||||
if ! cd "$(dirname "$0")"; then
|
||||
exit 125
|
||||
fi
|
||||
|
||||
# initialise counters
|
||||
TESTS=0
|
||||
FAILED=0
|
||||
|
|
@ -77,6 +82,17 @@ else
|
|||
PEER_GNUTLS=""
|
||||
fi
|
||||
|
||||
guess_config_name() {
|
||||
if git diff --quiet ../include/mbedtls/mbedtls_config.h 2>/dev/null; then
|
||||
echo "default"
|
||||
else
|
||||
echo "unknown"
|
||||
fi
|
||||
}
|
||||
: ${MBEDTLS_TEST_OUTCOME_FILE=}
|
||||
: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"}
|
||||
: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
|
||||
|
||||
# default values for options
|
||||
# /!\ keep this synchronised with:
|
||||
# - basic-build-test.sh
|
||||
|
|
@ -110,6 +126,8 @@ print_usage() {
|
|||
printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
|
||||
printf " -M|--memcheck\tCheck memory leaks and errors.\n"
|
||||
printf " -v|--verbose\tSet verbose output.\n"
|
||||
printf " --outcome-file\tFile where test outcomes are written\n"
|
||||
printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
|
||||
}
|
||||
|
||||
get_options() {
|
||||
|
|
@ -139,6 +157,9 @@ get_options() {
|
|||
-M|--memcheck)
|
||||
MEMCHECK=1
|
||||
;;
|
||||
--outcome-file)
|
||||
shift; MBEDTLS_TEST_OUTCOME_FILE=$1
|
||||
;;
|
||||
-h|--help)
|
||||
print_usage
|
||||
exit 0
|
||||
|
|
@ -805,6 +826,39 @@ wait_client_done() {
|
|||
echo "EXIT: $EXIT" >> $CLI_OUT
|
||||
}
|
||||
|
||||
# record_outcome <outcome> [<failure-reason>]
|
||||
record_outcome() {
|
||||
echo "$1"
|
||||
if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
|
||||
# The test outcome file has the format (in single line):
|
||||
# platform;configuration;
|
||||
# test suite name;test case description;
|
||||
# PASS/FAIL/SKIP;[failure cause]
|
||||
printf '%s;%s;%s;%s;%s;%s\n' \
|
||||
"$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
|
||||
"compat" "$TITLE" \
|
||||
"$1" "${2-}" \
|
||||
>> "$MBEDTLS_TEST_OUTCOME_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# display additional information if test case fails
|
||||
report_fail() {
|
||||
FAIL_PROMPT="outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
|
||||
record_outcome "FAIL" "$FAIL_PROMPT"
|
||||
cp $SRV_OUT c-srv-${TESTS}.log
|
||||
cp $CLI_OUT c-cli-${TESTS}.log
|
||||
echo " ! $FAIL_PROMPT"
|
||||
|
||||
if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
|
||||
echo " ! server output:"
|
||||
cat c-srv-${TESTS}.log
|
||||
echo " ! ==================================================="
|
||||
echo " ! client output:"
|
||||
cat c-cli-${TESTS}.log
|
||||
fi
|
||||
}
|
||||
|
||||
# run_client PROGRAM_NAME STANDARD_CIPHER_SUITE PROGRAM_CIPHER_SUITE
|
||||
run_client() {
|
||||
# announce what we're going to do
|
||||
|
|
@ -817,7 +871,7 @@ run_client() {
|
|||
# should we skip?
|
||||
if [ "X$SKIP_NEXT" = "XYES" ]; then
|
||||
SKIP_NEXT="NO"
|
||||
echo "SKIP"
|
||||
record_outcome "SKIP"
|
||||
SKIPPED=$(( $SKIPPED + 1 ))
|
||||
return
|
||||
fi
|
||||
|
|
@ -911,26 +965,14 @@ run_client() {
|
|||
# report and count result
|
||||
case $RESULT in
|
||||
"0")
|
||||
echo PASS
|
||||
record_outcome "PASS"
|
||||
;;
|
||||
"1")
|
||||
echo SKIP
|
||||
record_outcome "SKIP"
|
||||
SKIPPED=$(( $SKIPPED + 1 ))
|
||||
;;
|
||||
"2")
|
||||
echo FAIL
|
||||
cp $SRV_OUT c-srv-${TESTS}.log
|
||||
cp $CLI_OUT c-cli-${TESTS}.log
|
||||
echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
|
||||
|
||||
if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
|
||||
echo " ! server output:"
|
||||
cat c-srv-${TESTS}.log
|
||||
echo " ! ==================================================="
|
||||
echo " ! client output:"
|
||||
cat c-cli-${TESTS}.log
|
||||
fi
|
||||
|
||||
report_fail
|
||||
FAILED=$(( $FAILED + 1 ))
|
||||
;;
|
||||
esac
|
||||
|
|
@ -942,13 +984,16 @@ run_client() {
|
|||
# MAIN
|
||||
#
|
||||
|
||||
if cd $( dirname $0 ); then :; else
|
||||
echo "cd $( dirname $0 ) failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
get_options "$@"
|
||||
|
||||
# Make the outcome file path relative to the original directory, not
|
||||
# to .../tests
|
||||
case "$MBEDTLS_TEST_OUTCOME_FILE" in
|
||||
[!/]*)
|
||||
MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
|
||||
;;
|
||||
esac
|
||||
|
||||
# sanity checks, avoid an avalanche of errors
|
||||
if [ ! -x "$M_SRV" ]; then
|
||||
echo "Command '$M_SRV' is not an executable file" >&2
|
||||
|
|
|
|||
|
|
@ -177,6 +177,9 @@ get_options() {
|
|||
-p|--preserve-logs)
|
||||
PRESERVE_LOGS=1
|
||||
;;
|
||||
--outcome-file)
|
||||
shift; MBEDTLS_TEST_OUTCOME_FILE=$1
|
||||
;;
|
||||
--port)
|
||||
shift; SRV_PORT=$1
|
||||
;;
|
||||
|
|
@ -200,14 +203,6 @@ get_options() {
|
|||
done
|
||||
}
|
||||
|
||||
# Make the outcome file path relative to the original directory, not
|
||||
# to .../tests
|
||||
case "$MBEDTLS_TEST_OUTCOME_FILE" in
|
||||
[!/]*)
|
||||
MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Read boolean configuration options from mbedtls_config.h for easy and quick
|
||||
# testing. Skip non-boolean options (with something other than spaces
|
||||
# and a comment after "#define SYMBOL"). The variable contains a
|
||||
|
|
@ -1791,6 +1786,14 @@ cleanup() {
|
|||
|
||||
get_options "$@"
|
||||
|
||||
# Make the outcome file path relative to the original directory, not
|
||||
# to .../tests
|
||||
case "$MBEDTLS_TEST_OUTCOME_FILE" in
|
||||
[!/]*)
|
||||
MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
|
||||
;;
|
||||
esac
|
||||
|
||||
populate_enabled_hash_algs
|
||||
|
||||
# Optimize filters: if $FILTER and $EXCLUDE can be expressed as shell
|
||||
|
|
|
|||
Loading…
Reference in a new issue