diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eae1f5781..a8e7523e5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,6 +20,13 @@ endif() # on non-POSIX platforms. add_definitions("-D_POSIX_C_SOURCE=200809L") +# Test suites caught by SKIP_TEST_SUITES are built but not executed. +# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar" +# but not "test_suite_foobar". +string(REGEX REPLACE "[ ,;]" "|" SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES}") +string(REPLACE "." "\\." SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES_REGEX}") +set(SKIP_TEST_SUITES_REGEX "^(${SKIP_TEST_SUITES_REGEX})(\$|\\.)") + function(add_test_suite suite_name) if(ARGV1) set(data_name ${ARGV1}) @@ -36,7 +43,11 @@ function(add_test_suite suite_name) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_executable(test_suite_${data_name} test_suite_${data_name}.c) target_link_libraries(test_suite_${data_name} ${libs}) - add_test(${data_name}-suite test_suite_${data_name} --verbose) + if(${data_name} MATCHES ${SKIP_TEST_SUITES_REGEX}) + message(STATUS "The test suite ${data_name} will not be executed.") + else() + add_test(${data_name}-suite test_suite_${data_name} --verbose) + endif() endfunction(add_test_suite) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) @@ -52,6 +63,7 @@ endif(MSVC) add_test_suite(aes aes.ecb) add_test_suite(aes aes.cbc) add_test_suite(aes aes.cfb) +add_test_suite(aes aes.ofb) add_test_suite(aes aes.rest) add_test_suite(aes aes.xts) add_test_suite(arc4) diff --git a/tests/Makefile b/tests/Makefile index 78670c3fa..598bed6bd 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -121,8 +121,9 @@ ifneq ($(wildcard TESTS/.*),) endif endif +# Test suites caught by SKIP_TEST_SUITES are built but not executed. check: $(BINARIES) - perl scripts/run-test-suites.pl + perl scripts/run-test-suites.pl --skip=$(SKIP_TEST_SUITES) test: check diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index 058a203c3..329ed146a 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -4,19 +4,24 @@ # # This file is part of mbed TLS (https://tls.mbed.org) # -# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved -# -# Purpose -# -# Executes all the available test suites, and provides a basic summary of the -# results. -# -# Usage: run-test-suites.pl [-v] -# -# Options : -# -v|--verbose - Provide a pass/fail/skip breakdown per test suite and -# in total -# +# Copyright (c) 2015-2018, ARM Limited, All Rights Reserved + +=head1 SYNOPSIS + +Execute all the test suites and print a summary of the results. + + run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]] + +Options: + + -v|--verbose Print detailed failure information. + -v 2|--verbose=2 Print detailed failure information and summary messages. + -v 3|--verbose=3 Print detailed information about every test case. + --skip=SUITE[,SUITE...] + Skip the specified SUITE(s). This option can be used + multiple times. + +=cut use warnings; use strict; @@ -24,10 +29,15 @@ use strict; use utf8; use open qw(:std utf8); -use Getopt::Long; +use Getopt::Long qw(:config auto_help gnu_compat); +use Pod::Usage; my $verbose = 0; -GetOptions( "verbose|v:1" => \$verbose ); +my @skip_patterns = (); +GetOptions( + 'skip=s' => \@skip_patterns, + 'verbose|v:1' => \$verbose, + ) or die; # All test suites = executable files, excluding source files, debug # and profiling information, etc. We can't just grep {! /\./} because @@ -36,6 +46,17 @@ my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*'; @suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites; die "$0: no test suite found\n" unless @suites; +# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar" +# but not "test_suite_foobar". +my $skip_re = + ( '\Atest_suite_(' . + join('|', map { + s/[ ,;]/|/g; # allow any of " ,;|" as separators + s/\./\./g; # "." in the input means ".", not "any character" + $_ + } @skip_patterns) . + ')(\z|\.)' ); + # in case test suites are linked dynamically $ENV{'LD_LIBRARY_PATH'} = '../library:../crypto/library'; $ENV{'DYLD_LIBRARY_PATH'} = '../library:../crypto/library'; @@ -45,6 +66,7 @@ my $prefix = $^O eq "MSWin32" ? '' : './'; my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed, $suite_cases_failed, $suite_cases_skipped, $total_cases_passed, $total_cases_failed, $total_cases_skipped ); +my $suites_skipped = 0; sub pad_print_center { my( $width, $padchar, $string ) = @_; @@ -55,6 +77,12 @@ sub pad_print_center { for my $suite (@suites) { print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " "; + if( $suite =~ /$skip_re/o ) { + print "SKIP\n"; + ++$suites_skipped; + next; + } + my $command = "$prefix$suite"; if( $verbose ) { $command .= ' -v'; @@ -101,7 +129,10 @@ for my $suite (@suites) print "-" x 72, "\n"; print $failed_suites ? "FAILED" : "PASSED"; -printf " (%d suites, %d tests run)\n", scalar @suites, $total_tests_run; +printf( " (%d suites, %d tests run%s)\n", + scalar(@suites) - $suites_skipped, + $total_tests_run, + $suites_skipped ? ", $suites_skipped suites skipped" : "" ); if( $verbose > 1 ) { print " test cases passed :", $total_cases_passed, "\n"; @@ -111,8 +142,11 @@ if( $verbose > 1 ) { "\n"; print " of available tests :", ( $total_cases_passed + $total_cases_failed + $total_cases_skipped ), - "\n" + "\n"; + if( $suites_skipped != 0 ) { + print "Note: $suites_skipped suites were skipped.\n"; } +} exit( $failed_suites ? 1 : 0 );