From 51dcc24998298bffdfd58a42eaf8ff2c252222af Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2019 15:13:48 +0200 Subject: [PATCH] Test outcome file support: test suites If the environment variable MBEDTLS_TEST_OUTCOME_FILE is set, then for each test case, write a line to the file with the given name, of the form PLATFORM;CONFIGURATION;TEST SUITE;TEST CASE DESCRIPTION;PASS/FAIL/SKIP;CAUSE PLATFORM and CONFIGURATION come from the environment variables MBEDTLS_TEST_PLATFORM and MBEDTLS_TEST_CONFIGURATION. Errors while writing the test outcome file are not considered fatal, and are not reported except for an error initially opening the file. This is in line with other write errors that are not checked. --- tests/suites/host_test.function | 130 ++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/tests/suites/host_test.function b/tests/suites/host_test.function index d74af05bf..9e56ca3ed 100644 --- a/tests/suites/host_test.function +++ b/tests/suites/host_test.function @@ -368,6 +368,118 @@ static int run_test_snprintf( void ) test_snprintf( 5, "123", 3 ) != 0 ); } +/** \brief Write the description of the test case to the outcome CSV file. + * + * \param outcome_file The file to write to. + * If this is \c NULL, this function does nothing. + * \param argv0 The test suite name. + * \param test_case The test case description. + */ +static void write_outcome_entry( FILE *outcome_file, + const char *argv0, + const char *test_case ) +{ + /* The non-varying fields are initialized on first use. */ + static const char *platform = NULL; + static const char *configuration = NULL; + static const char *test_suite = NULL; + + if( outcome_file == NULL ) + return; + + if( platform == NULL ) + { + platform = getenv( "MBEDTLS_TEST_PLATFORM" ); + if( platform == NULL ) + platform = "unknown"; + } + if( configuration == NULL ) + { + configuration = getenv( "MBEDTLS_TEST_CONFIGURATION" ); + if( configuration == NULL ) + configuration = "unknown"; + } + if( test_suite == NULL ) + { + test_suite = strrchr( argv0, '/' ); + if( test_suite != NULL ) + test_suite += 1; // skip the '/' + else + test_suite = argv0; + } + + /* Write the beginning of the outcome line. + * Ignore errors: writing the outcome file is on a best-effort basis. */ + mbedtls_fprintf( outcome_file, "%s;%s;%s;%s;", + platform, configuration, test_suite, test_case ); +} + +/** \brief Write the result of the test case to the outcome CSV file. + * + * \param outcome_file The file to write to. + * If this is \c NULL, this function does nothing. + * \param unmet_dep_count The number of unmet dependencies. + * \param unmet_dependencies The array of unmet dependencies. + * \param ret The test dispatch status (DISPATCH_xxx). + * \param test_info A pointer to the test info structure. + */ +static void write_outcome_result( FILE *outcome_file, + size_t unmet_dep_count, + char *unmet_dependencies[], + int ret, + const test_info_t *info ) +{ + if( outcome_file == NULL ) + return; + + /* Write the end of the outcome line. + * Ignore errors: writing the outcome file is on a best-effort basis. */ + switch( ret ) + { + case DISPATCH_TEST_SUCCESS: + if( unmet_dep_count > 0 ) + { + size_t i; + mbedtls_fprintf( outcome_file, "SKIP" ); + for( i = 0; i < unmet_dep_count; i++ ) + { + mbedtls_fprintf( outcome_file, "%c%s", + i == 0 ? ';' : ':', + unmet_dependencies[i] ); + } + break; + } + switch( info->result ) + { + case TEST_RESULT_SUCCESS: + mbedtls_fprintf( outcome_file, "PASS;" ); + break; + case TEST_RESULT_SKIPPED: + mbedtls_fprintf( outcome_file, "SKIP;Runtime skip" ); + break; + default: + mbedtls_fprintf( outcome_file, "FAIL;%s:%d:%s", + info->filename, info->line_no, + info->test ); + break; + } + break; + case DISPATCH_TEST_FN_NOT_FOUND: + mbedtls_fprintf( outcome_file, "FAIL;Test function not found" ); + break; + case DISPATCH_INVALID_TEST_DATA: + mbedtls_fprintf( outcome_file, "FAIL;Invalid test data" ); + break; + case DISPATCH_UNSUPPORTED_SUITE: + mbedtls_fprintf( outcome_file, "SKIP;Unsupported suite" ); + break; + default: + mbedtls_fprintf( outcome_file, "FAIL;Unknown cause" ); + break; + } + mbedtls_fprintf( outcome_file, "\n" ); + fflush( outcome_file ); +} /** * \brief Desktop implementation of execute_tests(). @@ -404,6 +516,8 @@ int execute_tests( int argc , const char ** argv ) #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) int stdout_fd = -1; #endif /* __unix__ || __APPLE__ __MACH__ */ + const char *outcome_file_name = getenv( "MBEDTLS_TEST_OUTCOME_FILE" ); + FILE *outcome_file = NULL; #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \ !defined(TEST_SUITE_MEMORY_BUFFER_ALLOC) @@ -411,6 +525,15 @@ int execute_tests( int argc , const char ** argv ) mbedtls_memory_buffer_alloc_init( alloc_buf, sizeof( alloc_buf ) ); #endif + if( outcome_file_name != NULL ) + { + outcome_file = fopen( outcome_file_name, "a" ); + if( outcome_file == NULL ) + { + mbedtls_fprintf( stderr, "Unable to open outcome file. Continuing anyway.\n" ); + } + } + /* * The C standard doesn't guarantee that all-bits-0 is the representation * of a NULL pointer. We do however use that in our code for initializing @@ -506,6 +629,7 @@ int execute_tests( int argc , const char ** argv ) mbedtls_fprintf( stdout, "." ); mbedtls_fprintf( stdout, " " ); fflush( stdout ); + write_outcome_entry( outcome_file, argv[0], buf ); total_tests++; @@ -585,6 +709,9 @@ int execute_tests( int argc , const char ** argv ) } + write_outcome_result( outcome_file, + unmet_dep_count, unmet_dependencies, + ret, &test_info ); if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE ) { total_skipped++; @@ -653,6 +780,9 @@ int execute_tests( int argc , const char ** argv ) free( unmet_dependencies[i] ); } + if( outcome_file != NULL ) + fclose( outcome_file ); + mbedtls_fprintf( stdout, "\n----------------------------------------------------------------------------\n\n"); if( total_errors == 0 ) mbedtls_fprintf( stdout, "PASSED" );