Merge pull request #5090 from gilles-peskine-arm/ssl-opt-resend-retry-3.0
Retry a test case if it fails due to an unexpected resend
This commit is contained in:
paul-elliott-arm
GitHub
commit
cbe4a056bd
1 changed files with 184 additions and 104 deletions
288
tests/ssl-opt.sh
288
tests/ssl-opt.sh
|
|
@ -552,6 +552,32 @@ record_outcome() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# True if the presence of the given pattern in a log definitely indicates
|
||||||
|
# that the test has failed. False if the presence is inconclusive.
|
||||||
|
#
|
||||||
|
# Inputs:
|
||||||
|
# * $1: pattern found in the logs
|
||||||
|
# * $TIMES_LEFT: >0 if retrying is an option
|
||||||
|
#
|
||||||
|
# Outputs:
|
||||||
|
# * $outcome: set to a retry reason if the pattern is inconclusive,
|
||||||
|
# unchanged otherwise.
|
||||||
|
# * Return value: 1 if the pattern is inconclusive,
|
||||||
|
# 0 if the failure is definitive.
|
||||||
|
log_pattern_presence_is_conclusive() {
|
||||||
|
# If we've run out of attempts, then don't retry no matter what.
|
||||||
|
if [ $TIMES_LEFT -eq 0 ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
case $1 in
|
||||||
|
"resend")
|
||||||
|
# An undesired resend may have been caused by the OS dropping or
|
||||||
|
# delaying a packet at an inopportune time.
|
||||||
|
outcome="RETRY(resend)"
|
||||||
|
return 1;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
# fail <message>
|
# fail <message>
|
||||||
fail() {
|
fail() {
|
||||||
record_outcome "FAIL" "$1"
|
record_outcome "FAIL" "$1"
|
||||||
|
|
@ -805,68 +831,12 @@ skip_handshake_stage_check() {
|
||||||
SKIP_HANDSHAKE_CHECK="YES"
|
SKIP_HANDSHAKE_CHECK="YES"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Usage: run_test name [-p proxy_cmd] srv_cmd cli_cmd cli_exit [option [...]]
|
# Analyze the commands that will be used in a test.
|
||||||
# Options: -s pattern pattern that must be present in server output
|
#
|
||||||
# -c pattern pattern that must be present in client output
|
# Analyze and possibly instrument $PXY_CMD, $CLI_CMD, $SRV_CMD to pass
|
||||||
# -u pattern lines after pattern must be unique in client output
|
# extra arguments or go through wrappers.
|
||||||
# -f call shell function on client output
|
# Set $DTLS (0=TLS, 1=DTLS).
|
||||||
# -S pattern pattern that must be absent in server output
|
analyze_test_commands() {
|
||||||
# -C pattern pattern that must be absent in client output
|
|
||||||
# -U pattern lines after pattern must be unique in server output
|
|
||||||
# -F call shell function on server output
|
|
||||||
# -g call shell function on server and client output
|
|
||||||
run_test() {
|
|
||||||
NAME="$1"
|
|
||||||
shift 1
|
|
||||||
|
|
||||||
if is_excluded "$NAME"; then
|
|
||||||
SKIP_NEXT="NO"
|
|
||||||
# There was no request to run the test, so don't record its outcome.
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
print_name "$NAME"
|
|
||||||
|
|
||||||
# Do we only run numbered tests?
|
|
||||||
if [ -n "$RUN_TEST_NUMBER" ]; then
|
|
||||||
case ",$RUN_TEST_NUMBER," in
|
|
||||||
*",$TESTS,"*) :;;
|
|
||||||
*) SKIP_NEXT="YES";;
|
|
||||||
esac
|
|
||||||
fi
|
|
||||||
|
|
||||||
# does this test use a proxy?
|
|
||||||
if [ "X$1" = "X-p" ]; then
|
|
||||||
PXY_CMD="$2"
|
|
||||||
shift 2
|
|
||||||
else
|
|
||||||
PXY_CMD=""
|
|
||||||
fi
|
|
||||||
|
|
||||||
# get commands and client output
|
|
||||||
SRV_CMD="$1"
|
|
||||||
CLI_CMD="$2"
|
|
||||||
CLI_EXPECT="$3"
|
|
||||||
shift 3
|
|
||||||
|
|
||||||
# Check if test uses files
|
|
||||||
case "$SRV_CMD $CLI_CMD" in
|
|
||||||
*data_files/*)
|
|
||||||
requires_config_enabled MBEDTLS_FS_IO;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# If the client or serve requires a ciphersuite, check that it's enabled.
|
|
||||||
maybe_requires_ciphersuite_enabled "$SRV_CMD" "$@"
|
|
||||||
maybe_requires_ciphersuite_enabled "$CLI_CMD" "$@"
|
|
||||||
|
|
||||||
# should we skip?
|
|
||||||
if [ "X$SKIP_NEXT" = "XYES" ]; then
|
|
||||||
SKIP_NEXT="NO"
|
|
||||||
record_outcome "SKIP"
|
|
||||||
SKIPS=$(( $SKIPS + 1 ))
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
# update DTLS variable
|
# update DTLS variable
|
||||||
detect_dtls "$SRV_CMD"
|
detect_dtls "$SRV_CMD"
|
||||||
|
|
||||||
|
|
@ -920,48 +890,29 @@ run_test() {
|
||||||
CLI_CMD="valgrind --leak-check=full $CLI_CMD"
|
CLI_CMD="valgrind --leak-check=full $CLI_CMD"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
TIMES_LEFT=2
|
# Check for failure conditions after a test case.
|
||||||
while [ $TIMES_LEFT -gt 0 ]; do
|
#
|
||||||
TIMES_LEFT=$(( $TIMES_LEFT - 1 ))
|
# Inputs from run_test:
|
||||||
|
# * positional parameters: test options (see run_test documentation)
|
||||||
|
# * $CLI_EXIT: client return code
|
||||||
|
# * $CLI_EXPECT: expected client return code
|
||||||
|
# * $SRV_RET: server return code
|
||||||
|
# * $CLI_OUT, $SRV_OUT, $PXY_OUT: files containing client/server/proxy logs
|
||||||
|
# * $TIMES_LEFT: if nonzero, a RETRY outcome is allowed
|
||||||
|
#
|
||||||
|
# Outputs:
|
||||||
|
# * $outcome: one of PASS/RETRY*/FAIL
|
||||||
|
check_test_failure() {
|
||||||
|
outcome=FAIL
|
||||||
|
|
||||||
# run the commands
|
if [ $TIMES_LEFT -gt 0 ] &&
|
||||||
if [ -n "$PXY_CMD" ]; then
|
grep '===CLIENT_TIMEOUT===' $CLI_OUT >/dev/null
|
||||||
printf "# %s\n%s\n" "$NAME" "$PXY_CMD" > $PXY_OUT
|
then
|
||||||
$PXY_CMD >> $PXY_OUT 2>&1 &
|
outcome="RETRY(client-timeout)"
|
||||||
PXY_PID=$!
|
return
|
||||||
wait_proxy_start "$PXY_PORT" "$PXY_PID"
|
fi
|
||||||
fi
|
|
||||||
|
|
||||||
check_osrv_dtls
|
|
||||||
printf '# %s\n%s\n' "$NAME" "$SRV_CMD" > $SRV_OUT
|
|
||||||
provide_input | $SRV_CMD >> $SRV_OUT 2>&1 &
|
|
||||||
SRV_PID=$!
|
|
||||||
wait_server_start "$SRV_PORT" "$SRV_PID"
|
|
||||||
|
|
||||||
printf '# %s\n%s\n' "$NAME" "$CLI_CMD" > $CLI_OUT
|
|
||||||
eval "$CLI_CMD" >> $CLI_OUT 2>&1 &
|
|
||||||
wait_client_done
|
|
||||||
|
|
||||||
sleep 0.05
|
|
||||||
|
|
||||||
# terminate the server (and the proxy)
|
|
||||||
kill $SRV_PID
|
|
||||||
wait $SRV_PID
|
|
||||||
SRV_RET=$?
|
|
||||||
|
|
||||||
if [ -n "$PXY_CMD" ]; then
|
|
||||||
kill $PXY_PID >/dev/null 2>&1
|
|
||||||
wait $PXY_PID
|
|
||||||
fi
|
|
||||||
|
|
||||||
# retry only on timeouts
|
|
||||||
if grep '===CLIENT_TIMEOUT===' $CLI_OUT >/dev/null; then
|
|
||||||
printf "RETRY "
|
|
||||||
else
|
|
||||||
TIMES_LEFT=0
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# check if the client and server went at least to the handshake stage
|
# check if the client and server went at least to the handshake stage
|
||||||
# (useful to avoid tests with only negative assertions and non-zero
|
# (useful to avoid tests with only negative assertions and non-zero
|
||||||
|
|
@ -1024,14 +975,18 @@ run_test() {
|
||||||
|
|
||||||
"-S")
|
"-S")
|
||||||
if grep -v '^==' $SRV_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
|
if grep -v '^==' $SRV_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
|
||||||
fail "pattern '$2' MUST NOT be present in the Server output"
|
if log_pattern_presence_is_conclusive "$2"; then
|
||||||
|
fail "pattern '$2' MUST NOT be present in the Server output"
|
||||||
|
fi
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
|
||||||
"-C")
|
"-C")
|
||||||
if grep -v '^==' $CLI_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
|
if grep -v '^==' $CLI_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
|
||||||
fail "pattern '$2' MUST NOT be present in the Client output"
|
if log_pattern_presence_is_conclusive "$2"; then
|
||||||
|
fail "pattern '$2' MUST NOT be present in the Client output"
|
||||||
|
fi
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
|
@ -1095,6 +1050,131 @@ run_test() {
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# if we're here, everything is ok
|
# if we're here, everything is ok
|
||||||
|
outcome=PASS
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run the current test case: start the server and if applicable the proxy, run
|
||||||
|
# the client, wait for all processes to finish or time out.
|
||||||
|
#
|
||||||
|
# Inputs:
|
||||||
|
# * $NAME: test case name
|
||||||
|
# * $CLI_CMD, $SRV_CMD, $PXY_CMD: commands to run
|
||||||
|
# * $CLI_OUT, $SRV_OUT, $PXY_OUT: files to contain client/server/proxy logs
|
||||||
|
#
|
||||||
|
# Outputs:
|
||||||
|
# * $CLI_EXIT: client return code
|
||||||
|
# * $SRV_RET: server return code
|
||||||
|
do_run_test_once() {
|
||||||
|
# run the commands
|
||||||
|
if [ -n "$PXY_CMD" ]; then
|
||||||
|
printf "# %s\n%s\n" "$NAME" "$PXY_CMD" > $PXY_OUT
|
||||||
|
$PXY_CMD >> $PXY_OUT 2>&1 &
|
||||||
|
PXY_PID=$!
|
||||||
|
wait_proxy_start "$PXY_PORT" "$PXY_PID"
|
||||||
|
fi
|
||||||
|
|
||||||
|
check_osrv_dtls
|
||||||
|
printf '# %s\n%s\n' "$NAME" "$SRV_CMD" > $SRV_OUT
|
||||||
|
provide_input | $SRV_CMD >> $SRV_OUT 2>&1 &
|
||||||
|
SRV_PID=$!
|
||||||
|
wait_server_start "$SRV_PORT" "$SRV_PID"
|
||||||
|
|
||||||
|
printf '# %s\n%s\n' "$NAME" "$CLI_CMD" > $CLI_OUT
|
||||||
|
eval "$CLI_CMD" >> $CLI_OUT 2>&1 &
|
||||||
|
wait_client_done
|
||||||
|
|
||||||
|
sleep 0.05
|
||||||
|
|
||||||
|
# terminate the server (and the proxy)
|
||||||
|
kill $SRV_PID
|
||||||
|
wait $SRV_PID
|
||||||
|
SRV_RET=$?
|
||||||
|
|
||||||
|
if [ -n "$PXY_CMD" ]; then
|
||||||
|
kill $PXY_PID >/dev/null 2>&1
|
||||||
|
wait $PXY_PID
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Usage: run_test name [-p proxy_cmd] srv_cmd cli_cmd cli_exit [option [...]]
|
||||||
|
# Options: -s pattern pattern that must be present in server output
|
||||||
|
# -c pattern pattern that must be present in client output
|
||||||
|
# -u pattern lines after pattern must be unique in client output
|
||||||
|
# -f call shell function on client output
|
||||||
|
# -S pattern pattern that must be absent in server output
|
||||||
|
# -C pattern pattern that must be absent in client output
|
||||||
|
# -U pattern lines after pattern must be unique in server output
|
||||||
|
# -F call shell function on server output
|
||||||
|
# -g call shell function on server and client output
|
||||||
|
run_test() {
|
||||||
|
NAME="$1"
|
||||||
|
shift 1
|
||||||
|
|
||||||
|
if is_excluded "$NAME"; then
|
||||||
|
SKIP_NEXT="NO"
|
||||||
|
# There was no request to run the test, so don't record its outcome.
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
print_name "$NAME"
|
||||||
|
|
||||||
|
# Do we only run numbered tests?
|
||||||
|
if [ -n "$RUN_TEST_NUMBER" ]; then
|
||||||
|
case ",$RUN_TEST_NUMBER," in
|
||||||
|
*",$TESTS,"*) :;;
|
||||||
|
*) SKIP_NEXT="YES";;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
# does this test use a proxy?
|
||||||
|
if [ "X$1" = "X-p" ]; then
|
||||||
|
PXY_CMD="$2"
|
||||||
|
shift 2
|
||||||
|
else
|
||||||
|
PXY_CMD=""
|
||||||
|
fi
|
||||||
|
|
||||||
|
# get commands and client output
|
||||||
|
SRV_CMD="$1"
|
||||||
|
CLI_CMD="$2"
|
||||||
|
CLI_EXPECT="$3"
|
||||||
|
shift 3
|
||||||
|
|
||||||
|
# Check if test uses files
|
||||||
|
case "$SRV_CMD $CLI_CMD" in
|
||||||
|
*data_files/*)
|
||||||
|
requires_config_enabled MBEDTLS_FS_IO;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# If the client or serve requires a ciphersuite, check that it's enabled.
|
||||||
|
maybe_requires_ciphersuite_enabled "$SRV_CMD" "$@"
|
||||||
|
maybe_requires_ciphersuite_enabled "$CLI_CMD" "$@"
|
||||||
|
|
||||||
|
# should we skip?
|
||||||
|
if [ "X$SKIP_NEXT" = "XYES" ]; then
|
||||||
|
SKIP_NEXT="NO"
|
||||||
|
record_outcome "SKIP"
|
||||||
|
SKIPS=$(( $SKIPS + 1 ))
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
analyze_test_commands "$@"
|
||||||
|
|
||||||
|
TIMES_LEFT=2
|
||||||
|
while [ $TIMES_LEFT -gt 0 ]; do
|
||||||
|
TIMES_LEFT=$(( $TIMES_LEFT - 1 ))
|
||||||
|
|
||||||
|
do_run_test_once
|
||||||
|
|
||||||
|
check_test_failure "$@"
|
||||||
|
case $outcome in
|
||||||
|
PASS) break;;
|
||||||
|
RETRY*) printf "$outcome ";;
|
||||||
|
FAIL) return;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# If we get this far, the test case passed.
|
||||||
record_outcome "PASS"
|
record_outcome "PASS"
|
||||||
if [ "$PRESERVE_LOGS" -gt 0 ]; then
|
if [ "$PRESERVE_LOGS" -gt 0 ]; then
|
||||||
mv $SRV_OUT o-srv-${TESTS}.log
|
mv $SRV_OUT o-srv-${TESTS}.log
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue