diff --git a/tests/compat.sh b/tests/compat.sh index 724311cf7..06243bde4 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -645,14 +645,14 @@ setup_arguments() exit 1; esac - P_SERVER_ARGS="server_addr=0.0.0.0 force_version=$MODE" - O_SERVER_ARGS="-www -cipher NULL,ALL -$MODE" - G_SERVER_ARGS="-p 4433 --http" + P_SERVER_ARGS="server_port=$PORT server_addr=0.0.0.0 force_version=$MODE" + O_SERVER_ARGS="-accept $PORT -www -cipher NULL,ALL -$MODE" + G_SERVER_ARGS="-p $PORT --http" G_SERVER_PRIO="EXPORT:+NULL:+MD5:+PSK:+DHE-PSK:+ECDHE-PSK:+RSA-PSK:-VERS-TLS-ALL:$G_PRIO_MODE" - P_CLIENT_ARGS="force_version=$MODE" - O_CLIENT_ARGS="-$MODE" - G_CLIENT_ARGS="-p 4433 --debug 3" + P_CLIENT_ARGS="server_port=$PORT force_version=$MODE" + O_CLIENT_ARGS="-connect localhost:$PORT -$MODE" + G_CLIENT_ARGS="-p $PORT --debug 3" G_CLIENT_PRIO="NONE:$G_PRIO_MODE:+COMP-NULL:+CURVE-ALL:+SIGN-ALL" if [ "X$VERIFY" = "XYES" ]; @@ -757,8 +757,8 @@ start_server() { SERVER_NAME=$1 log "$SERVER_CMD" - echo "$SERVER_CMD" > srv_out - $SERVER_CMD >> srv_out 2>&1 & + echo "$SERVER_CMD" > $SRV_OUT + $SERVER_CMD >> $SRV_OUT 2>&1 & PROCESS_ID=$! sleep 1 @@ -768,6 +768,10 @@ start_server() { stop_server() { case $SERVER_NAME in [Pp]olar*) + # start watchdog in case SERVERQUIT fails + ( sleep 20; echo "SERVERQUIT TIMEOUT"; kill $MAIN_PID ) & + WATCHDOG_PID=$! + # we must force a PSK suite when in PSK mode (otherwise client # auth will fail), so try every entry in $P_CIPHERS in turn (in # case the first one is not implemented in this configuration) @@ -779,27 +783,31 @@ stop_server() { break fi done + + wait $PROCESS_ID 2>/dev/null + kill $WATCHDOG_PID 2>/dev/null + wait $WATCHDOG_PID 2>/dev/null ;; *) kill $PROCESS_ID 2>/dev/null + wait $PROCESS_ID 2>/dev/null esac - wait $PROCESS_ID 2>/dev/null if [ "$MEMCHECK" -gt 0 ]; then - if is_polar "$SERVER_CMD" && has_mem_err srv_out; then + if is_polar "$SERVER_CMD" && has_mem_err $SRV_OUT; then echo " ! Server had memory errors" let "srvmem++" return fi fi - rm -f srv_out + rm -f $SRV_OUT } # kill the running server (used when killed by signal) cleanup() { - rm -f srv_out cli_out + rm -f $SRV_OUT $CLI_OUT kill $PROCESS_ID exit 1 } @@ -820,14 +828,14 @@ run_client() { [Oo]pen*) CLIENT_CMD="$OPENSSL_CMD s_client $O_CLIENT_ARGS -cipher $2" log "$CLIENT_CMD" - echo "$CLIENT_CMD" > cli_out - ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> cli_out 2>&1 + echo "$CLIENT_CMD" > $CLI_OUT + ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> $CLI_OUT 2>&1 EXIT=$? if [ "$EXIT" == "0" ]; then RESULT=0 else - if grep 'Cipher is (NONE)' cli_out >/dev/null; then + if grep 'Cipher is (NONE)' $CLI_OUT >/dev/null; then RESULT=1 else RESULT=2 @@ -838,8 +846,8 @@ run_client() { [Gg]nu*) CLIENT_CMD="$GNUTLS_CLI $G_CLIENT_ARGS --priority $G_PRIO_MODE:$2 localhost" log "$CLIENT_CMD" - echo "$CLIENT_CMD" > cli_out - ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> cli_out 2>&1 + echo "$CLIENT_CMD" > $CLI_OUT + ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> $CLI_OUT 2>&1 EXIT=$? if [ "$EXIT" == "0" ]; then @@ -848,8 +856,8 @@ run_client() { RESULT=2 # interpret early failure, with a handshake_failure alert # before the server hello, as "no ciphersuite in common" - if grep -F 'Received alert [40]: Handshake failed' cli_out; then - if grep -i 'SERVER HELLO .* was received' cli_out; then : + if grep -F 'Received alert [40]: Handshake failed' $CLI_OUT; then + if grep -i 'SERVER HELLO .* was received' $CLI_OUT; then : else RESULT=1 fi @@ -863,8 +871,8 @@ run_client() { CLIENT_CMD="valgrind --leak-check=full $CLIENT_CMD" fi log "$CLIENT_CMD" - echo "$CLIENT_CMD" > cli_out - $CLIENT_CMD >> cli_out 2>&1 + echo "$CLIENT_CMD" > $CLI_OUT + $CLIENT_CMD >> $CLI_OUT 2>&1 EXIT=$? case $EXIT in @@ -874,7 +882,7 @@ run_client() { esac if [ "$MEMCHECK" -gt 0 ]; then - if is_polar "$CLIENT_CMD" && has_mem_err cli_out; then + if is_polar "$CLIENT_CMD" && has_mem_err $CLI_OUT; then RESULT=2 fi fi @@ -887,7 +895,7 @@ run_client() { ;; esac - echo "EXIT: $EXIT" >> cli_out + echo "EXIT: $EXIT" >> $CLI_OUT # report and count result case $RESULT in @@ -900,14 +908,14 @@ run_client() { ;; "2") echo FAIL - cp srv_out c-srv-${tests}.log - cp cli_out c-cli-${tests}.log + cp $SRV_OUT c-srv-${tests}.log + cp $CLI_OUT c-cli-${tests}.log echo " ! outputs saved to c-srv-${tests}.log, c-cli-${tests}.log" let "failed++" ;; esac - rm -f cli_out + rm -f $CLI_OUT } # @@ -952,7 +960,17 @@ for PEER in $PEERS; do esac done -killall -q gnutls-serv openssl ssl_server ssl_server2 +# used by watchdog +MAIN_PID="$$" + +# Pick a "unique" port in the range 10000-19999. +PORT="0000$$" +PORT="1$(echo $PORT | tail -c 4)" + +# Also pick a unique name for intermediate files +SRV_OUT="srv_out.$$" +CLI_OUT="cli_out.$$" + trap cleanup INT TERM HUP for VERIFY in $VERIFIES; do diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 1e11d78c8..d01dc259e 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -6,8 +6,8 @@ # CMake configuration. After this script is run, the CMake cache is lost and # CMake is not initialised any more! # -# Assumes gcc, clang (recent enough for using ASan) are available, as weel as -# cmake. Also assumes valgrind is available if --memcheck is used. +# Assumes gcc and clang (recent enough for using ASan) are available, +# as well as cmake and valgrind. # Abort on errors (and uninitiliased variables) set -eu @@ -54,54 +54,72 @@ msg() echo "******************************************************************" } -# Step 1: various build types +# The test ordering tries to optimize for the following criteria: +# 1. Catch possible problems early, by running first test that run quickly +# and/or are more likely to fail than others. +# 2. Minimize total running time, by avoiding useless rebuilds +# +# Indicative running times are given for reference. -msg "Unix make, default compiler and flags" -cleanup -make - -msg "cmake, gcc with lots of warnings" +msg "build: cmake, gcc with lots of warnings" # ~ 1 min cleanup CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Check . make -msg "cmake, clang with lots of warnings" +msg "test: main suites with valgrind" # ~ 2 min 10s +make memcheck + +msg "build: with ASan" # ~ 1 min +cleanup +cmake -D CMAKE_BUILD_TYPE:String=ASan . +make + +msg "test: ssl-opt.sh (ASan build)" # ~ 1 min 10s +cd tests +./ssl-opt.sh +cd .. + +msg "test: main suites and selftest (ASan build)" # ~ 10s + 30s +make test +programs/test/selftest + +msg "test: ref-configs (ASan build)" # ~ 4 min 45 s +tests/scripts/test-ref-configs.pl + +# Most issues are likely to be caught at this point + +msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min +make + +msg "test: compat.sh (ASan build)" # ~ 7 min 30s +cd tests +./compat.sh +cd .. + +msg "build: cmake, clang with lots of warnings" # ~ 40s cleanup CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check . make -# Step 2: Full tests, with ASan - -msg "ASan build and full tests" +msg "build: Unix make, -O2" # ~ 30s cleanup -cmake -D CMAKE_BUILD_TYPE:String=ASan . make -make test -programs/test/selftest -cd tests -./compat.sh -./ssl-opt.sh -cd .. -tests/scripts/test-ref-configs.pl -# Step 3: using valgrind's memcheck - -msg "Release build, test suites with valgrind's memcheck" -cleanup -# optimized build to compensate a bit for valgrind slowdown -cmake -D CMAKE_BUILD_TYPE:String=Release . -make -make memcheck +# Optional parts that take a long time to run if [ "$MEMORY" -gt 0 ]; then + msg "test: ssl-opt --memcheck (-02 build)" # ~ 8 min cd tests ./ssl-opt.sh --memcheck - [ "$MEMORY" -gt 1 ] && ./compat.sh --memcheck cd .. - # no test-ref-configs: doesn't have a memcheck option (yet?) -fi -# Done + if [ "$MEMORY" -gt 1 ]; then + msg "test: compat --memcheck (-02 build)" # ~ 42 min + cd tests + ./compat.sh --memcheck + cd .. + fi +fi echo "Done." cleanup diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index f4fbc019b..238995ff8 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -89,8 +89,8 @@ fail() { echo "FAIL" echo " ! $1" - cp srv_out o-srv-${TESTS}.log - cp cli_out o-cli-${TESTS}.log + cp $SRV_OUT o-srv-${TESTS}.log + cp $CLI_OUT o-cli-${TESTS}.log echo " ! outputs saved to o-srv-${TESTS}.log and o-cli-${TESTS}.log" FAILS=`echo $FAILS + 1 | bc` @@ -112,6 +112,28 @@ has_mem_err() { fi } +# wait for server to be ready +wait_srv_ready() { + if is_polar "$SRV_CMD"; then + READY_MSG="Waiting for a remote connection" + else + READY_MSG="ACCEPT" + fi + + # If the server isn't ready after 10 secs, something probably went wrong + ( sleep 10; echo "SERVERSTART TIMEOUT"; kill $MAIN_PID ) & + WATCHDOG_PID=$! + + while ! grep "$READY_MSG" $SRV_OUT >/dev/null; do + # don't use sleep, since the whole goal is to avoid wasting time, + # and 1 second is usually way more than the server needs to start + true + done + + kill $WATCHDOG_PID + wait $WATCHDOG_PID +} + # Usage: run_test name srv_cmd cli_cmd cli_exit [option [...]] # Options: -s pattern pattern that must be present in server output # -c pattern pattern that must be present in client output @@ -142,38 +164,46 @@ run_test() { fi # run the commands - echo "$SRV_CMD" > srv_out - $SRV_CMD >> srv_out 2>&1 & + echo "$SRV_CMD" > $SRV_OUT + $SRV_CMD >> $SRV_OUT 2>&1 & SRV_PID=$! - sleep 1 - echo "$CLI_CMD" > cli_out - eval "$CLI_CMD" >> cli_out 2>&1 + wait_srv_ready + echo "$CLI_CMD" > $CLI_OUT + eval "$CLI_CMD" >> $CLI_OUT 2>&1 CLI_EXIT=$? - echo "EXIT: $CLI_EXIT" >> cli_out + echo "EXIT: $CLI_EXIT" >> $CLI_OUT - # psk is useful when server only has bad certs if is_polar "$SRV_CMD"; then - "$P_CLI" request_page=SERVERQUIT tickets=0 auth_mode=none psk=abc123 \ + # start watchdog in case SERVERQUIT fails + ( sleep 10; echo "SERVERQUIT TIMEOUT"; kill $MAIN_PID ) & + WATCHDOG_PID=$! + + # psk is useful when server only has bad certs + $P_CLI request_page=SERVERQUIT tickets=0 auth_mode=none psk=abc123 \ crt_file=data_files/cli2.crt key_file=data_files/cli2.key \ >/dev/null + + wait $SRV_PID + kill $WATCHDOG_PID + wait $WATCHDOG_PID else kill $SRV_PID + wait $SRV_PID fi - wait $SRV_PID # check if the client and server went at least to the handshake stage # (useful to avoid tests with only negative assertions and non-zero # expected client exit to incorrectly succeed in case of catastrophic # failure) if is_polar "$SRV_CMD"; then - if grep "Performing the SSL/TLS handshake" srv_out >/dev/null; then :; + if grep "Performing the SSL/TLS handshake" $SRV_OUT >/dev/null; then :; else fail "server failed to start" return fi fi if is_polar "$CLI_CMD"; then - if grep "Performing the SSL/TLS handshake" cli_out >/dev/null; then :; + if grep "Performing the SSL/TLS handshake" $CLI_OUT >/dev/null; then :; else fail "client failed to start" return @@ -199,28 +229,28 @@ run_test() { do case $1 in "-s") - if grep "$2" srv_out >/dev/null; then :; else + if grep "$2" $SRV_OUT >/dev/null; then :; else fail "-s $2" return fi ;; "-c") - if grep "$2" cli_out >/dev/null; then :; else + if grep "$2" $CLI_OUT >/dev/null; then :; else fail "-c $2" return fi ;; "-S") - if grep "$2" srv_out >/dev/null; then + if grep "$2" $SRV_OUT >/dev/null; then fail "-S $2" return fi ;; "-C") - if grep "$2" cli_out >/dev/null; then + if grep "$2" $CLI_OUT >/dev/null; then fail "-C $2" return fi @@ -235,11 +265,11 @@ run_test() { # check valgrind's results if [ "$MEMCHECK" -gt 0 ]; then - if is_polar "$SRV_CMD" && has_mem_err srv_out; then + if is_polar "$SRV_CMD" && has_mem_err $SRV_OUT; then fail "Server has memory errors" return fi - if is_polar "$CLI_CMD" && has_mem_err cli_out; then + if is_polar "$CLI_CMD" && has_mem_err $CLI_OUT; then fail "Client has memory errors" return fi @@ -247,11 +277,11 @@ run_test() { # if we're here, everything is ok echo "PASS" - rm -f srv_out cli_out + rm -f $SRV_OUT $CLI_OUT } cleanup() { - rm -f cli_out srv_out sess + rm -f $CLI_OUT $SRV_OUT $SESSION kill $SRV_PID exit 1 } @@ -276,7 +306,24 @@ if which $OPENSSL_CMD >/dev/null 2>&1; then :; else exit 1 fi -killall -q openssl ssl_server ssl_server2 +# used by watchdog +MAIN_PID="$$" + +# Pick a "unique" port in the range 10000-19999. +PORT="0000$$" +PORT="1$(echo $PORT | tail -c 4)" + +# fix commands to use this port +P_SRV="$P_SRV server_port=$PORT" +P_CLI="$P_CLI server_port=$PORT" +O_SRV="$O_SRV -accept $PORT" +O_CLI="$O_CLI -connect localhost:$PORT" + +# Also pick a unique name for intermediate files +SRV_OUT="srv_out.$$" +CLI_OUT="cli_out.$$" +SESSION="session.$$" + trap cleanup INT TERM HUP # Test for SSLv2 ClientHello @@ -365,7 +412,9 @@ run_test "Session resume using tickets #4 (openssl server)" \ run_test "Session resume using tickets #5 (openssl client)" \ "$P_SRV debug_level=4 tickets=1" \ - "($O_CLI -sess_out sess; $O_CLI -sess_in sess; rm -f sess)" \ + "( $O_CLI -sess_out $SESSION; \ + $O_CLI -sess_in $SESSION; \ + rm -f $SESSION )" \ 0 \ -s "found session ticket extension" \ -s "server hello, adding session ticket extension" \ @@ -450,7 +499,9 @@ run_test "Session resume using cache #7 (no timeout)" \ run_test "Session resume using cache #8 (openssl client)" \ "$P_SRV debug_level=4 tickets=0" \ - "($O_CLI -sess_out sess; $O_CLI -sess_in sess; rm -f sess)" \ + "( $O_CLI -sess_out $SESSION; \ + $O_CLI -sess_in $SESSION; \ + rm -f $SESSION )" \ 0 \ -s "found session ticket extension" \ -S "server hello, adding session ticket extension" \