From 0b2423403c1233c0848530dc06cf1ba36707e484 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 17 Feb 2016 10:11:21 +0000 Subject: [PATCH 01/10] x509: trailing bytes in DER: add integration tests --- tests/data_files/server5-der0.crt | Bin 0 -> 547 bytes tests/data_files/server5-der1a.crt | Bin 0 -> 548 bytes tests/data_files/server5-der1b.crt | Bin 0 -> 548 bytes tests/data_files/server5-der2.crt | Bin 0 -> 549 bytes tests/data_files/server5-der4.crt | Bin 0 -> 551 bytes tests/data_files/server5-der8.crt | Bin 0 -> 555 bytes tests/data_files/server5-der9.crt | Bin 0 -> 556 bytes tests/ssl-opt.sh | 58 +++++++++++++++++++++++++++++ 8 files changed, 58 insertions(+) create mode 100644 tests/data_files/server5-der0.crt create mode 100644 tests/data_files/server5-der1a.crt create mode 100644 tests/data_files/server5-der1b.crt create mode 100644 tests/data_files/server5-der2.crt create mode 100644 tests/data_files/server5-der4.crt create mode 100644 tests/data_files/server5-der8.crt create mode 100644 tests/data_files/server5-der9.crt diff --git a/tests/data_files/server5-der0.crt b/tests/data_files/server5-der0.crt new file mode 100644 index 0000000000000000000000000000000000000000..08d8dd311b525fd51171a1019ad3194dad91580a GIT binary patch literal 547 zcmXqLVv;v#VqCg_nTe5!iILNQi;Y98&EuRc3p0~}ogudYCmVAp3!5;LpO2xS0Y8Yt zCCm|!pOaV=9PDE#V;}_*Vipz#3l$gVD1@XImngV8D>yqE$cghB8XH&|nHZXy8X1^G ziSrtPxJFQ}feAtLg$x8B=5yxcCnx4)dUQv4h>n#0YgPGb1~*69bF+nXsE> zoN`e`cE=-i|10FZtNF<`vE;&9k*(h|lp>QR`8{R0p)C0SmHs7@*jTZ>T^)zA%Xvf3 zc4_hbVmz_s?f=D%a}642fxRp%%)(^AU?2;$U6zkUj720MacTb*_M6w67;`VyonyI+c^Rf1A}TbXwv-X(%>vG8}Y%RF~v@ z<^^)(FlR6rq%s*Y%+iUuzU=m*rzyN2cKY4Do_I~z@c8QDhTWGh7Ym21ox~lx`of;? z>-3*3RMa$`y2{Ry$w1Mpe(tf@W8ACNKdH)Ee?0%uR8{2p(~r})Mm~-ctx4NCq53$Z RfPe8Z2E`yPzK_SR0ss?|s)_&r literal 0 HcmV?d00001 diff --git a/tests/data_files/server5-der1a.crt b/tests/data_files/server5-der1a.crt new file mode 100644 index 0000000000000000000000000000000000000000..015017b17db1c360392790665896ea46dc0feac2 GIT binary patch literal 548 zcmXqLVv;v#VqCg_nTe5!iILNQi;Y98&EuRc3p0~}ogudYCmVAp3!5;LpO2xS0Y8Yt zCCm|!pOaV=9PDE#V;}_*Vipz#3l$gVD1@XImngV8D>yqE$cghB8XH&|nHZXy8X1^G ziSrtPxJFQ}feAtLg$x8B=5yxcCnx4)dUQv4h>n#0YgPGb1~*69bF+nXsE> zoN`e`cE=-i|10FZtNF<`vE;&9k*(h|lp>QR`8{R0p)C0SmHs7@*jTZ>T^)zA%Xvf3 zc4_hbVmz_s?f=D%a}642fxRp%%)(^AU?2;$U6zkUj720MacTb*_M6w67;`VyonyI+c^Rf1A}TbXwv-X(%>vG8}Y%RF~v@ z<^^)(FlR6rq%s*Y%+iUuzU=m*rzyN2cKY4Do_I~z@c8QDhTWGh7Ym21ox~lx`of;? z>-3*3RMa$`y2{Ry$w1Mpe(tf@W8ACNKdH)Ee?0%uR8{2p(~r})Mm~-ctx4NCq53$Z SfPe8Z2E`yPzK_SRG5`R+9IA={ literal 0 HcmV?d00001 diff --git a/tests/data_files/server5-der1b.crt b/tests/data_files/server5-der1b.crt new file mode 100644 index 0000000000000000000000000000000000000000..6340d9e2ed9fb5e60822f52182c08cddf98f4417 GIT binary patch literal 548 zcmXqLVv;v#VqCg_nTe5!iILNQi;Y98&EuRc3p0~}ogudYCmVAp3!5;LpO2xS0Y8Yt zCCm|!pOaV=9PDE#V;}_*Vipz#3l$gVD1@XImngV8D>yqE$cghB8XH&|nHZXy8X1^G ziSrtPxJFQ}feAtLg$x8B=5yxcCnx4)dUQv4h>n#0YgPGb1~*69bF+nXsE> zoN`e`cE=-i|10FZtNF<`vE;&9k*(h|lp>QR`8{R0p)C0SmHs7@*jTZ>T^)zA%Xvf3 zc4_hbVmz_s?f=D%a}642fxRp%%)(^AU?2;$U6zkUj720MacTb*_M6w67;`VyonyI+c^Rf1A}TbXwv-X(%>vG8}Y%RF~v@ z<^^)(FlR6rq%s*Y%+iUuzU=m*rzyN2cKY4Do_I~z@c8QDhTWGh7Ym21ox~lx`of;? z>-3*3RMa$`y2{Ry$w1Mpe(tf@W8ACNKdH)Ee?0%uR8{2p(~r})Mm~-ctx4NCq53$Z SfPe8Z2E`yPzK_SR9s~fs-K$Fg literal 0 HcmV?d00001 diff --git a/tests/data_files/server5-der2.crt b/tests/data_files/server5-der2.crt new file mode 100644 index 0000000000000000000000000000000000000000..c6e320a369c20c3ee8c54d3caa1d5af0a7225206 GIT binary patch literal 549 zcmXqLVv;v#VqCg_nTe5!iILNQi;Y98&EuRc3p0~}ogudYCmVAp3!5;LpO2xS0Y8Yt zCCm|!pOaV=9PDE#V;}_*Vipz#3l$gVD1@XImngV8D>yqE$cghB8XH&|nHZXy8X1^G ziSrtPxJFQ}feAtLg$x8B=5yxcCnx4)dUQv4h>n#0YgPGb1~*69bF+nXsE> zoN`e`cE=-i|10FZtNF<`vE;&9k*(h|lp>QR`8{R0p)C0SmHs7@*jTZ>T^)zA%Xvf3 zc4_hbVmz_s?f=D%a}642fxRp%%)(^AU?2;$U6zkUj720MacTb*_M6w67;`VyonyI+c^Rf1A}TbXwv-X(%>vG8}Y%RF~v@ z<^^)(FlR6rq%s*Y%+iUuzU=m*rzyN2cKY4Do_I~z@c8QDhTWGh7Ym21ox~lx`of;? z>-3*3RMa$`y2{Ry$w1Mpe(tf@W8ACNKdH)Ee?0%uR8{2p(~r})Mm~-ctx4NCq53$Z TfPe8Z2E`yPzK_SR?&JahYB8%# literal 0 HcmV?d00001 diff --git a/tests/data_files/server5-der4.crt b/tests/data_files/server5-der4.crt new file mode 100644 index 0000000000000000000000000000000000000000..4af05cce1ed05ea02e9fac3fed3a0904b44799b0 GIT binary patch literal 551 zcmXqLVv;v#VqCg_nTe5!iILNQi;Y98&EuRc3p0~}ogudYCmVAp3!5;LpO2xS0Y8Yt zCCm|!pOaV=9PDE#V;}_*Vipz#3l$gVD1@XImngV8D>yqE$cghB8XH&|nHZXy8X1^G ziSrtPxJFQ}feAtLg$x8B=5yxcCnx4)dUQv4h>n#0YgPGb1~*69bF+nXsE> zoN`e`cE=-i|10FZtNF<`vE;&9k*(h|lp>QR`8{R0p)C0SmHs7@*jTZ>T^)zA%Xvf3 zc4_hbVmz_s?f=D%a}642fxRp%%)(^AU?2;$U6zkUj720MacTb*_M6w67;`VyonyI+c^Rf1A}TbXwv-X(%>vG8}Y%RF~v@ z<^^)(FlR6rq%s*Y%+iUuzU=m*rzyN2cKY4Do_I~z@c8QDhTWGh7Ym21ox~lx`of;? z>-3*3RMa$`y2{Ry$w1Mpe(tf@W8ACNKdH)Ee?0%uR8{2p(~r})Mm~-ctx4NCq53$Z VfPe8Z2E`yPzK_SRE*F>*4*yqE$cghB8XH&|nHZXy8X1^G ziSrtPxJFQ}feAtLg$x8B=5yxcCnx4)dUQv4h>n#0YgPGb1~*69bF+nXsE> zoN`e`cE=-i|10FZtNF<`vE;&9k*(h|lp>QR`8{R0p)C0SmHs7@*jTZ>T^)zA%Xvf3 zc4_hbVmz_s?f=D%a}642fxRp%%)(^AU?2;$U6zkUj720MacTb*_M6w67;`VyonyI+c^Rf1A}TbXwv-X(%>vG8}Y%RF~v@ z<^^)(FlR6rq%s*Y%+iUuzU=m*rzyN2cKY4Do_I~z@c8QDhTWGh7Ym21ox~lx`of;? z>-3*3RMa$`y2{Ry$w1Mpe(tf@W8ACNKdH)Ee?0%uR8{2p(~r})Mm~-ctx4NCq53$Z ZfPe8Z2E`yPzK_SRE?NFxU9D;rKLC6Lu2cX3 literal 0 HcmV?d00001 diff --git a/tests/data_files/server5-der9.crt b/tests/data_files/server5-der9.crt new file mode 100644 index 0000000000000000000000000000000000000000..4947f1f83fad41a48cee838ccf8cfdf2f2100e29 GIT binary patch literal 556 zcmXqLVv;v#VqCg_nTe5!iILNQi;Y98&EuRc3p0~}ogudYCmVAp3!5;LpO2xS0Y8Yt zCCm|!pOaV=9PDE#V;}_*Vipz#3l$gVD1@XImngV8D>yqE$cghB8XH&|nHZXy8X1^G ziSrtPxJFQ}feAtLg$x8B=5yxcCnx4)dUQv4h>n#0YgPGb1~*69bF+nXsE> zoN`e`cE=-i|10FZtNF<`vE;&9k*(h|lp>QR`8{R0p)C0SmHs7@*jTZ>T^)zA%Xvf3 zc4_hbVmz_s?f=D%a}642fxRp%%)(^AU?2;$U6zkUj720MacTb*_M6w67;`VyonyI+c^Rf1A}TbXwv-X(%>vG8}Y%RF~v@ z<^^)(FlR6rq%s*Y%+iUuzU=m*rzyN2cKY4Do_I~z@c8QDhTWGh7Ym21ox~lx`of;? z>-3*3RMa$`y2{Ry$w1Mpe(tf@W8ACNKdH)Ee?0%uR8{2p(~r})Mm~-ctx4NCq53$Z afPe8Z2E`yPzK_SRp8sKBT=suSl_mf!qOWiO literal 0 HcmV?d00001 diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index c0b6f94d6..e1ecbca33 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -1554,6 +1554,64 @@ run_test "Renego ext: gnutls client unsafe, server break legacy" \ -S "received TLS_EMPTY_RENEGOTIATION_INFO\|found renegotiation extension" \ -S "server hello, secure renegotiation extension" +# Tests for silently dropping trailing extra bytes in .der certificates + +requires_gnutls +run_test "DER format: no trailing bytes" \ + "$P_SRV crt_file=data_files/server5-der0.crt \ + key_file=data_files/server5.key" \ + "$G_CLI " \ + 0 \ + -c "Handshake was completed" \ + +requires_gnutls +run_test "DER format: with a trailing zero byte" \ + "$P_SRV crt_file=data_files/server5-der1a.crt \ + key_file=data_files/server5.key" \ + "$G_CLI " \ + 0 \ + -c "Handshake was completed" \ + +requires_gnutls +run_test "DER format: with a trailing random byte" \ + "$P_SRV crt_file=data_files/server5-der1b.crt \ + key_file=data_files/server5.key" \ + "$G_CLI " \ + 0 \ + -c "Handshake was completed" \ + +requires_gnutls +run_test "DER format: with 2 trailing random bytes" \ + "$P_SRV crt_file=data_files/server5-der2.crt \ + key_file=data_files/server5.key" \ + "$G_CLI " \ + 0 \ + -c "Handshake was completed" \ + +requires_gnutls +run_test "DER format: with 4 trailing random bytes" \ + "$P_SRV crt_file=data_files/server5-der4.crt \ + key_file=data_files/server5.key" \ + "$G_CLI " \ + 0 \ + -c "Handshake was completed" \ + +requires_gnutls +run_test "DER format: with 8 trailing random bytes" \ + "$P_SRV crt_file=data_files/server5-der8.crt \ + key_file=data_files/server5.key" \ + "$G_CLI " \ + 0 \ + -c "Handshake was completed" \ + +requires_gnutls +run_test "DER format: with 9 trailing random bytes" \ + "$P_SRV crt_file=data_files/server5-der9.crt \ + key_file=data_files/server5.key" \ + "$G_CLI " \ + 0 \ + -c "Handshake was completed" \ + # Tests for auth_mode run_test "Authentication: server badcert, client required" \ From e154f95e035ed07763a95ccda25bd7074454242b Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 17 Feb 2016 14:24:28 +0000 Subject: [PATCH 02/10] x509: trailing bytes in DER: correct a unit test One of the unit test was failing, because it was testing behavior that was part of the bug. Updated the return value to the correct one --- tests/suites/test_suite_x509parse.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/test_suite_x509parse.data b/tests/suites/test_suite_x509parse.data index 2f2137f54..6b04ae37e 100644 --- a/tests/suites/test_suite_x509parse.data +++ b/tests/suites/test_suite_x509parse.data @@ -755,7 +755,7 @@ X509 Certificate ASN1 (Incorrect first tag) x509parse_crt:"":"":MBEDTLS_ERR_X509_INVALID_FORMAT X509 Certificate ASN1 (Correct first tag, data length does not match) -x509parse_crt:"300000":"":MBEDTLS_ERR_X509_INVALID_FORMAT + MBEDTLS_ERR_ASN1_UNEXPECTED_TAG +x509parse_crt:"300000":"":MBEDTLS_ERR_X509_INVALID_FORMAT + MBEDTLS_ERR_ASN1_OUT_OF_DATA X509 Certificate ASN1 (Correct first tag, no more data) x509parse_crt:"3000":"":MBEDTLS_ERR_X509_INVALID_FORMAT + MBEDTLS_ERR_ASN1_OUT_OF_DATA From cc0e49ddde3d8dbfcbbdc725b3d48482fb8015f1 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 17 Feb 2016 14:34:12 +0000 Subject: [PATCH 03/10] x509: trailing bytes in DER: fix bug Fix bug in mbedtls_x509_crt_parse that caused trailing extra data in the buffer after DER certificates to be included in the raw representation. #377 --- ChangeLog | 2 ++ library/x509_crt.c | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index e9b67908f..ed32f0b37 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,8 @@ Bugfix * Fix issue in Makefile that prevented building using armar. #386 * Fix memory leak that occured only when ECJPAKE was enabled and ECDHE and ECDSA was disabled in config.h . The leak didn't occur by default. + * Fix bug in mbedtls_x509_crt_parse that caused trailing extra data in the + buffer after DER certificates to be included in the raw representation. Changes * On ARM platforms, when compiling with -O0 with GCC, Clang or armcc5, diff --git a/library/x509_crt.c b/library/x509_crt.c index 6dc5ad34f..a1ce2544e 100644 --- a/library/x509_crt.c +++ b/library/x509_crt.c @@ -680,14 +680,9 @@ static int x509_crt_parse_der_core( mbedtls_x509_crt *crt, const unsigned char * if( crt == NULL || buf == NULL ) return( MBEDTLS_ERR_X509_BAD_INPUT_DATA ); - p = mbedtls_calloc( 1, len = buflen ); - if( p == NULL ) - return( MBEDTLS_ERR_X509_ALLOC_FAILED ); - - memcpy( p, buf, buflen ); - - crt->raw.p = p; - crt->raw.len = len; + // Use the original buffer until we figure out actual length + p = (unsigned char*) buf; + len = buflen; end = p + len; /* @@ -711,6 +706,18 @@ static int x509_crt_parse_der_core( mbedtls_x509_crt *crt, const unsigned char * } crt_end = p + len; + // Create and populate a new buffer for the raw field + crt->raw.len = crt_end - buf; + crt->raw.p = p = mbedtls_calloc( 1, crt->raw.len ); + if( p == NULL ) + return( MBEDTLS_ERR_X509_ALLOC_FAILED ); + + memcpy( p, buf, crt->raw.len ); + + // Direct pointers to the new buffer + p += crt->raw.len - len; + end = crt_end = p + len; + /* * TBSCertificate ::= SEQUENCE { */ From f413b6fffe23da366920ca73d428303deac6acfd Mon Sep 17 00:00:00 2001 From: Simon Butcher Date: Mon, 14 Mar 2016 22:32:42 +0000 Subject: [PATCH 04/10] Fix to stop all.sh corrupting config.h The test script all.sh was persisting the SSL3 configuration in config.h through more tests than intended and not restoring the config the end. --- tests/scripts/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 467f22a93..1cc82562c 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -133,6 +133,7 @@ tests/compat.sh msg "build: Default + SSLv3 (ASan build)" # ~ 6 min cleanup +cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl set MBEDTLS_SSL_PROTO_SSL3 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -149,7 +150,6 @@ tests/ssl-opt.sh msg "build: cmake, full config, clang" # ~ 50s cleanup -cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check . From 5a8afb848a76eadd272e2d717f6bd065e546401b Mon Sep 17 00:00:00 2001 From: SimonB Date: Fri, 11 Mar 2016 00:40:54 +0000 Subject: [PATCH 05/10] Fix exit code and add a count of the test suites Now counts and displays the number of test suites executed, which can vary depending on build configurations. All tests are now executed as this is a sample and test program, rather than exit on first failure. Exit code now restricted to SUCCESS or FAILURE. --- programs/test/selftest.c | 161 +++++++++++++++++++++++++++++++-------- 1 file changed, 129 insertions(+), 32 deletions(-) diff --git a/programs/test/selftest.c b/programs/test/selftest.c index fe5d51426..5c4c737f3 100644 --- a/programs/test/selftest.c +++ b/programs/test/selftest.c @@ -99,7 +99,8 @@ static int run_test_snprintf( void ) int main( int argc, char *argv[] ) { - int ret = 0, v; + int ret = 0, v, suites_tested = 0, suites_failed =0, + exitcode = EXIT_SUCCESS; #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) unsigned char buf[1000000]; #endif @@ -126,8 +127,11 @@ int main( int argc, char *argv[] ) return( 0 ); } - if( argc == 2 && strcmp( argv[1], "-quiet" ) == 0 ) + if( argc == 2 && ( strcmp( argv[1], "--quiet" ) == 0 || + strcmp( argv[1], "-q" ) == 0 ) ) + { v = 0; + } else { v = 1; @@ -142,134 +146,212 @@ int main( int argc, char *argv[] ) #if defined(MBEDTLS_MD2_C) if( ( ret = mbedtls_md2_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_MD4_C) if( ( ret = mbedtls_md4_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_MD5_C) if( ( ret = mbedtls_md5_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_RIPEMD160_C) if( ( ret = mbedtls_ripemd160_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_SHA1_C) if( ( ret = mbedtls_sha1_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_SHA256_C) if( ( ret = mbedtls_sha256_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_SHA512_C) if( ( ret = mbedtls_sha512_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_ARC4_C) if( ( ret = mbedtls_arc4_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_DES_C) if( ( ret = mbedtls_des_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_AES_C) if( ( ret = mbedtls_aes_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_GCM_C) && defined(MBEDTLS_AES_C) if( ( ret = mbedtls_gcm_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_CCM_C) && defined(MBEDTLS_AES_C) if( ( ret = mbedtls_ccm_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_BASE64_C) if( ( ret = mbedtls_base64_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_BIGNUM_C) if( ( ret = mbedtls_mpi_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_RSA_C) if( ( ret = mbedtls_rsa_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_X509_USE_C) if( ( ret = mbedtls_x509_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_XTEA_C) if( ( ret = mbedtls_xtea_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_CAMELLIA_C) if( ( ret = mbedtls_camellia_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_CTR_DRBG_C) if( ( ret = mbedtls_ctr_drbg_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_HMAC_DRBG_C) if( ( ret = mbedtls_hmac_drbg_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_ECP_C) if( ( ret = mbedtls_ecp_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_ECJPAKE_C) if( ( ret = mbedtls_ecjpake_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_DHM_C) if( ( ret = mbedtls_dhm_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_ENTROPY_C) if( ( ret = mbedtls_entropy_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #if defined(MBEDTLS_PKCS5_C) if( ( ret = mbedtls_pkcs5_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif /* Slow tests last */ #if defined(MBEDTLS_TIMING_C) if( ( ret = mbedtls_timing_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif #else @@ -285,19 +367,34 @@ int main( int argc, char *argv[] ) #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) mbedtls_memory_buffer_alloc_free(); - if( ( ret = mbedtls_memory_buffer_alloc_self_test( v ) ) != 0 ) - return( ret ); + { + suites_failed++; + } + suites_tested++; #endif if( v != 0 ) { - mbedtls_printf( " [ All tests passed ]\n\n" ); + mbedtls_printf( " Executed %d test suites\n\n", suites_tested); + + if( suites_failed > 0) + { + mbedtls_printf( " [ %d tests FAIL ]\n\n", suites_failed ); + } + else + { + mbedtls_printf( " [ All tests PASS ]\n\n" ); + } #if defined(_WIN32) mbedtls_printf( " Press Enter to exit this program.\n" ); fflush( stdout ); getchar(); #endif } - return( ret ); + if( suites_failed > 0) + exitcode = EXIT_FAILURE; + + exit(exitcode); } + From ad8fbc066c2f3d05ba8564ab1496f511c9de38bf Mon Sep 17 00:00:00 2001 From: SimonB Date: Fri, 11 Mar 2016 17:33:39 +0000 Subject: [PATCH 06/10] Add test result breakdown to test suites script Added a --verbose switch to 'run-test-suite.pl' to summarise the pass/fail/skip results of each test suite, and summary for all executed tests. --- tests/scripts/run-test-suites.pl | 65 ++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) mode change 100644 => 100755 tests/scripts/run-test-suites.pl diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl old mode 100644 new mode 100755 index b91355d30..0ac2a30fc --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -1,11 +1,36 @@ #!/usr/bin/perl +# run-test-suites.pl +# +# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved +# +# Purpose +# +# Executes all the available test suites, and provides a basic summary of the +# results. +# +# Usage: run-test-suites.pl [-v] +# +# Options : +# -v|--verbose - Provide a pass/fail/skip breakdown per test suite and +# in total +# + use warnings; use strict; use utf8; use open qw(:std utf8); +use constant FALSE => 0; +use constant TRUE => 1; + +my $verbose; +my $switch = shift; +if ( defined($switch) && ( $switch eq "-v" || $switch eq "--verbose" ) ) { + $verbose = TRUE; +} + my @suites = grep { ! /\.(?:c|gcno)$/ } glob 'test_suite_*'; die "$0: no test suite found\n" unless @suites; @@ -14,22 +39,56 @@ $ENV{'LD_LIBRARY_PATH'} = '../library'; my $prefix = $^O eq "MSWin32" ? '' : './'; -my ($failed_suites, $total_tests_run); +my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed, + $suite_cases_failed, $suite_cases_skipped, $total_cases_passed, + $total_cases_failed, $total_cases_skipped ); + for my $suite (@suites) { print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " "; my $result = `$prefix$suite`; + + $suite_cases_passed = () = $result =~ /.. PASS/g; + $suite_cases_failed = () = $result =~ /.. FAILED/g; + $suite_cases_skipped = () = $result =~ /.. ----/g; + if( $result =~ /PASSED/ ) { print "PASS\n"; - my ($tests, $skipped) = $result =~ /([0-9]*) tests.*?([0-9]*) skipped/; - $total_tests_run += $tests - $skipped; } else { $failed_suites++; print "FAIL\n"; } + + my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/; + $total_tests_run += $tests - $skipped; + + if ( $verbose ) { + print "(test cases passed:", $suite_cases_passed, + " failed:", $suite_cases_failed, + " skipped:", $suite_cases_skipped, + " of total:", ( $suite_cases_passed + $suite_cases_failed ), + ")\n" + } + + $total_cases_passed += $suite_cases_passed; + $total_cases_failed += $suite_cases_failed; + $total_cases_skipped += $suite_cases_skipped; } print "-" x 72, "\n"; print $failed_suites ? "FAILED" : "PASSED"; printf " (%d suites, %d tests run)\n", scalar @suites, $total_tests_run; + +if ( $verbose ) { + print " test cases passed :", $total_cases_passed, "\n"; + print " failed :", $total_cases_failed, "\n"; + print " skipped :", $total_cases_skipped, "\n"; + print " of tests executed :", ( $total_cases_passed + $total_cases_failed ), + "\n"; + print " of available tests :", + ( $total_cases_passed + $total_cases_failed + $total_cases_skipped ), + "\n" + } + exit( $failed_suites ? 1 : 0 ); + From 75f3caa4084e89a0d9eb54cee6ee5d16f32a4b9b Mon Sep 17 00:00:00 2001 From: SimonB Date: Sat, 12 Mar 2016 19:06:56 +0000 Subject: [PATCH 07/10] Fix unit test script to ignore coverage data The script run-test-suite.pl was mistaking .gcda code coverage data files as test suites. The files are now ignored. --- tests/scripts/run-test-suites.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index 0ac2a30fc..ed3aaab33 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -31,7 +31,7 @@ if ( defined($switch) && ( $switch eq "-v" || $switch eq "--verbose" ) ) { $verbose = TRUE; } -my @suites = grep { ! /\.(?:c|gcno)$/ } glob 'test_suite_*'; +my @suites = grep { ! /\.(?:c|gcno|gcda)$/ } glob 'test_suite_*'; die "$0: no test suite found\n" unless @suites; # in case test suites are linked dynamically From 21ab9d7b8b6cb601e7f5074f719f672762384f58 Mon Sep 17 00:00:00 2001 From: SimonB Date: Sat, 12 Mar 2016 20:37:32 +0000 Subject: [PATCH 08/10] Add a script to execute the basic tests basic-build-test.sh executes the most obvious and common test suites and creates a test report including coverage data. --- tests/scripts/basic-build-test.sh | 201 ++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100755 tests/scripts/basic-build-test.sh diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh new file mode 100755 index 000000000..3cc0b18f4 --- /dev/null +++ b/tests/scripts/basic-build-test.sh @@ -0,0 +1,201 @@ +#!/bin/sh + +# basic-build-tests.sh +# +# Copyright (c) 2016, ARM Limited, All Rights Reserved +# +# Purpose +# +# Executes the basic test suites, captures the results, and generates a simple +# test report and code coverage report. +# +# The tests include: +# * Self-tests - executed using program/test/selftest +# * Unit tests - executed using tests/scripts/run-test-suite.pl +# * System tests - executed using tests/ssl-opt.sh +# * Interoperability tests - executed using tests/compat.sh +# +# The tests focus on functionality and do not consider performance. +# +# Note the tests self-adapt due to configurations in include/mbedtls/config.h +# which can lead to some tests being skipped, and can cause the number of +# available self-tests to fluctuate. +# +# This script has been written to be generic and should work on any shell. +# +# Usage: basic-build-tests.sh +# + +# Abort on errors (and uninitiliased variables) +set -eu + +if [ -d library -a -d include -a -d tests ]; then :; else + echo "Must be run from mbed TLS root" >&2 + exit 1 +fi + + +# Step 1 - Make and instrumented build for code coverage +CFLAGS=' --coverage -g3 -O0 ' +make + + +# Step 2 - Execute the tests +TEST_OUTPUT=out_${PPID} +cd tests + +# Step 2a - Self-tests +../programs/test/selftest |tee self-test-$TEST_OUTPUT +echo + +# Step 2b - Unit Tests +perl scripts/run-test-suites.pl -v |tee unit-test-$TEST_OUTPUT +echo + +# Step 2c - System Tests +sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT +echo + +# Step 2d - Compatibility tests +sh compat.sh |tee compat-test-$TEST_OUTPUT +echo + +# Step 3 - Process the coverage report +cd .. +make lcov |tee tests/cov-$TEST_OUTPUT + + +# Step 4 - Summarise the test report +echo +echo "=========================================================================" +echo "Test Report Summary" +echo + +cd tests + +# Step 4a - Self-tests +echo "Self tests - ./programs/test/selftest" + +PASSED_TESTS=$(grep 'passed' self-test-$TEST_OUTPUT |wc -l) +FAILED_TESTS=$(grep 'failed' self-test-$TEST_OUTPUT |wc -l) +AVAIL_TESTS=$(($PASSED_TESTS + $FAILED_TESTS)) +EXED_TESTS=$(($PASSED_TESTS + $FAILED_TESTS)) + +echo "Passed : $PASSED_TESTS" +echo "Failed : $FAILED_TESTS" +echo "Skipped : n/a" +echo "Total tests : $AVAIL_TESTS" +echo + +TOTAL_PASS=$PASSED_TESTS +TOTAL_FAIL=$FAILED_TESTS +TOTAL_SKIP=0 +TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS)) +TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS)) + + +# Step 3b - Unit tests +echo "Unit tests - tests/scripts/run-test-suites.pl" + +PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ') +SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ') +TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ') +FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ') + +echo "No test suites : $TOTAL_SUITES" +echo "Passed : $PASSED_TESTS" +echo "Failed : $FAILED_TESTS" +echo "Skipped : $SKIPPED_TESTS" +echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))" +echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))" +echo + +TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) +TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) +TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) +TOTAL_AVAIL=$(($TOTAL_AVAIL + $PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) +TOTAL_EXED=$(($TOTAL_EXED + $PASSED_TESTS + $FAILED_TESTS)) + + +# Step 3c - System tests +echo "System tests - tests/ssl-opt.sh" + +PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') +SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') +TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p') +FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS)) + +echo "Passed : $PASSED_TESTS" +echo "Failed : $FAILED_TESTS" +echo "Skipped : $SKIPPED_TESTS" +echo "Total exec'd tests : $TOTAL_TESTS" +echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))" +echo + +TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) +TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) +TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) +TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) +TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) + + +# Step 3d - Compatibility tests +echo "Compatibility tests - tests/compat.sh" + +PASSED_TESTS=$(tail -n5 compat-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') +SKIPPED_TESTS=$(tail -n5 compat-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') +EXED_TESTS=$(tail -n5 compat-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p') +FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS)) + +echo "Passed : $PASSED_TESTS" +echo "Failed : $FAILED_TESTS" +echo "Skipped : $SKIPPED_TESTS" +echo "Total exec'd tests : $EXED_TESTS" +echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))" +echo + +TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) +TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) +TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) +TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS)) +TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS)) + + +# Step 3e - Grand totals +echo "-------------------------------------------------------------------------" +echo "Total tests" + +echo "Total Passed : $TOTAL_PASS" +echo "Total Failed : $TOTAL_FAIL" +echo "Total Skipped : $TOTAL_SKIP" +echo "Total exec'd tests : $TOTAL_EXED" +echo "Total avail tests : $TOTAL_AVAIL" +echo + + +# Step 3f - Coverage +echo "Coverage" + +LINES_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* lines)/\1/p') +LINES_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) lines)/\1/p') +FUNCS_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p') +FUNCS_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9.]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p') + +LINES_PERCENT=$((1000*$LINES_TESTED/$LINES_TOTAL)) +LINES_PERCENT="$(($LINES_PERCENT/10)).$(($LINES_PERCENT-($LINES_PERCENT/10)*10))" + +FUNCS_PERCENT=$((1000*$FUNCS_TESTED/$FUNCS_TOTAL)) +FUNCS_PERCENT="$(($FUNCS_PERCENT/10)).$(($FUNCS_PERCENT-($FUNCS_PERCENT/10)*10))" + +echo "Lines Tested : $LINES_TESTED of $LINES_TOTAL $LINES_PERCENT%" +echo "Functions Tested : $FUNCS_TESTED of $FUNCS_TOTAL $FUNCS_PERCENT%" +echo + + +rm self-test-$TEST_OUTPUT +rm unit-test-$TEST_OUTPUT +rm sys-test-$TEST_OUTPUT +rm compat-test-$TEST_OUTPUT +rm cov-$TEST_OUTPUT + +cd .. From ab0c51d782343f51d6b3ec9ada1f1d1dab899b53 Mon Sep 17 00:00:00 2001 From: Simon Butcher Date: Sun, 13 Mar 2016 01:23:34 +0000 Subject: [PATCH 09/10] Fix minor issues with basic test script Following fixes: * In the test script, 'basic-build-test.sh', the total number of functions had a broken RE, and was picking up the number of tested functions. * Titles of tests was misleading * The 'run-test-suites.pl' script was mistaking dSYM directories as test suites to be executed. --- tests/scripts/basic-build-test.sh | 10 +++++----- tests/scripts/run-test-suites.pl | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 3cc0b18f4..25248d622 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -117,8 +117,8 @@ TOTAL_AVAIL=$(($TOTAL_AVAIL + $PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) TOTAL_EXED=$(($TOTAL_EXED + $PASSED_TESTS + $FAILED_TESTS)) -# Step 3c - System tests -echo "System tests - tests/ssl-opt.sh" +# Step 3c - TLS Options tests +echo "TLS Options tests - tests/ssl-opt.sh" PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') @@ -139,8 +139,8 @@ TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) -# Step 3d - Compatibility tests -echo "Compatibility tests - tests/compat.sh" +# Step 3d - System Compatibility tests +echo "System/Compatibility tests - tests/compat.sh" PASSED_TESTS=$(tail -n5 compat-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') SKIPPED_TESTS=$(tail -n5 compat-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') @@ -179,7 +179,7 @@ echo "Coverage" LINES_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* lines)/\1/p') LINES_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) lines)/\1/p') FUNCS_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p') -FUNCS_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9.]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p') +FUNCS_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) functions)$/\1/p') LINES_PERCENT=$((1000*$LINES_TESTED/$LINES_TOTAL)) LINES_PERCENT="$(($LINES_PERCENT/10)).$(($LINES_PERCENT-($LINES_PERCENT/10)*10))" diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index ed3aaab33..fb77e1571 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -31,7 +31,7 @@ if ( defined($switch) && ( $switch eq "-v" || $switch eq "--verbose" ) ) { $verbose = TRUE; } -my @suites = grep { ! /\.(?:c|gcno|gcda)$/ } glob 'test_suite_*'; +my @suites = grep { ! /\.(?:c|gcno|gcda|dSYM)$/ } glob 'test_suite_*'; die "$0: no test suite found\n" unless @suites; # in case test suites are linked dynamically From f1547632dc318b14d7edc66d0cc59ed778be237d Mon Sep 17 00:00:00 2001 From: Simon Butcher Date: Mon, 14 Mar 2016 23:09:39 +0000 Subject: [PATCH 10/10] Fixes to style following review Made code spacing consistent with guidelines, and corrected the misnamed test steps in basic-build-test.sh --- programs/test/selftest.c | 6 +++--- tests/scripts/basic-build-test.sh | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/programs/test/selftest.c b/programs/test/selftest.c index 5c4c737f3..b168b7112 100644 --- a/programs/test/selftest.c +++ b/programs/test/selftest.c @@ -99,7 +99,7 @@ static int run_test_snprintf( void ) int main( int argc, char *argv[] ) { - int ret = 0, v, suites_tested = 0, suites_failed =0, + int ret = 0, v, suites_tested = 0, suites_failed = 0, exitcode = EXIT_SUCCESS; #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) unsigned char buf[1000000]; @@ -376,7 +376,7 @@ int main( int argc, char *argv[] ) if( v != 0 ) { - mbedtls_printf( " Executed %d test suites\n\n", suites_tested); + mbedtls_printf( " Executed %d test suites\n\n", suites_tested ); if( suites_failed > 0) { @@ -395,6 +395,6 @@ int main( int argc, char *argv[] ) if( suites_failed > 0) exitcode = EXIT_FAILURE; - exit(exitcode); + exit( exitcode ); } diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 25248d622..06c2eb9bd 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -94,7 +94,7 @@ TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS)) TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS)) -# Step 3b - Unit tests +# Step 4b - Unit tests echo "Unit tests - tests/scripts/run-test-suites.pl" PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ') @@ -117,7 +117,7 @@ TOTAL_AVAIL=$(($TOTAL_AVAIL + $PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) TOTAL_EXED=$(($TOTAL_EXED + $PASSED_TESTS + $FAILED_TESTS)) -# Step 3c - TLS Options tests +# Step 4c - TLS Options tests echo "TLS Options tests - tests/ssl-opt.sh" PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') @@ -139,7 +139,7 @@ TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) -# Step 3d - System Compatibility tests +# Step 4d - System Compatibility tests echo "System/Compatibility tests - tests/compat.sh" PASSED_TESTS=$(tail -n5 compat-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') @@ -161,7 +161,7 @@ TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS)) TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS)) -# Step 3e - Grand totals +# Step 4e - Grand totals echo "-------------------------------------------------------------------------" echo "Total tests" @@ -173,7 +173,7 @@ echo "Total avail tests : $TOTAL_AVAIL" echo -# Step 3f - Coverage +# Step 4f - Coverage echo "Coverage" LINES_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* lines)/\1/p')