Merge branch 'development' into development-restricted

* development: (22 commits)
  Only pass -Wformat-signedness to versions of GCC that support it.
  Documentation improvements
  Remove redundant assignment
  Add comments explaining include paths
  Library files aren't supposed to be executable
  CMake: Include the library directory for the sake of 3rdparty
  Factor common library properties
  Fix erroneous skip of test cases for disabled ciphersuites
  Include common.h instead of config.h in library source files
  Include the library directory for the sake of 3rdparty
  Fix copypasta in test case descriptions
  Remove metadata tests for features that are not implemented
  Fix dependency in PSA test cases
  Fix dependency in AES GCM test case
  Document the fields of TestCasesOutcomes
  Check test case coverage
  New script for test outcome analysis
  check_test_cases: move some functions into the logical class
  check_test_cases: move "walk" functions into a class
  check_test_cases: parametrize iteration functions by the action
  ...

3rdparty/everest/library/Hacl_Curve25519_joined.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED)
 

3rdparty/everest/library/everest.c

@@ -19,11 +19,7 @@
  *  This file is part of Mbed TLS (https://tls.mbed.org).
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #include <string.h>
 

3rdparty/everest/library/x25519.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ECDH_C) && defined(MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED)
 

CMakeLists.txt

@@ -165,7 +165,10 @@ if(CMAKE_COMPILER_IS_GNU)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wlogical-op")
     endif()
     if (GCC_VERSION VERSION_GREATER 4.8 OR GCC_VERSION VERSION_EQUAL 4.8)
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wshadow -Wformat-signedness")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wshadow")
+    endif()
+    if (GCC_VERSION VERSION_GREATER 5.0)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wformat-signedness")
     endif()
     set(CMAKE_C_FLAGS_RELEASE     "-O2")
     set(CMAKE_C_FLAGS_DEBUG       "-O0 -g3")

ChangeLog.d/cmake-install.txt

@@ -0,0 +1,3 @@
+Bugfix
+   * Library files installed after a CMake build no longer have execute
+     permission.

ChangeLog.d/format-signedness.txt

@@ -0,0 +1,3 @@
+Changes
+   * Only pass -Wformat-signedness to versions of GCC that support it. Reported
+     in #3478 and fix contributed in #3479 by okhowang.

docs/architecture/testing/test-framework.md

@@ -22,7 +22,7 @@ Each test case has a description which succinctly describes for a human audience
 * Make the description descriptive. “foo: x=2, y=4” is more descriptive than “foo #2”. “foo: 0<x<y, both even” is even better if these inequalities and parities are why this particular test data was chosen.
 * Avoid changing the description of an existing test case without a good reason. This breaks the tracking of failures across CI runs, since this tracking is based on the descriptions.
 
-`tests/scripts/check-test-cases.py` enforces some rules and warns if some guidelines are violated.
+`tests/scripts/check_test_cases.py` enforces some rules and warns if some guidelines are violated.
 
 ## TLS tests
 
@@ -32,7 +32,7 @@ Each test case has a description which succinctly describes for a human audience
 
 Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`.
 
-The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check-test-cases.py`.
+The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check_test_cases.py`.
 
 ## Running tests
 

library/CMakeLists.txt

@@ -148,10 +148,14 @@ if (NOT USE_STATIC_MBEDTLS_LIBRARY AND NOT USE_SHARED_MBEDTLS_LIBRARY)
     message(FATAL_ERROR "Need to choose static or shared mbedtls build!")
 endif(NOT USE_STATIC_MBEDTLS_LIBRARY AND NOT USE_SHARED_MBEDTLS_LIBRARY)
 
+set(target_libraries "mbedcrypto" "mbedx509" "mbedtls")
+
 if(USE_STATIC_MBEDTLS_LIBRARY AND USE_SHARED_MBEDTLS_LIBRARY)
     set(mbedtls_static_target "mbedtls_static")
     set(mbedx509_static_target "mbedx509_static")
     set(mbedcrypto_static_target "mbedcrypto_static")
+    list(APPEND target_libraries
+        "mbedcrypto_static" "mbedx509_static" "mbedtls_static")
 elseif(USE_STATIC_MBEDTLS_LIBRARY)
     set(mbedtls_static_target "mbedtls")
     set(mbedx509_static_target "mbedx509")
@@ -162,59 +166,48 @@ if(USE_STATIC_MBEDTLS_LIBRARY)
     add_library(${mbedcrypto_static_target} STATIC ${src_crypto})
     set_target_properties(${mbedcrypto_static_target} PROPERTIES OUTPUT_NAME mbedcrypto)
     target_link_libraries(${mbedcrypto_static_target} ${libs})
-    target_include_directories(${mbedcrypto_static_target}
-        PUBLIC ${MBEDTLS_DIR}/include/
-        PUBLIC ${thirdparty_inc_public}
-        PRIVATE ${thirdparty_inc})
-    target_compile_definitions(${mbedcrypto_static_target}
-        PRIVATE ${thirdparty_def})
 
     add_library(${mbedx509_static_target} STATIC ${src_x509})
     set_target_properties(${mbedx509_static_target} PROPERTIES OUTPUT_NAME mbedx509)
     target_link_libraries(${mbedx509_static_target} ${libs} ${mbedcrypto_static_target})
-    target_include_directories(${mbedx509_static_target}
-        PUBLIC ${MBEDTLS_DIR}/include/)
 
     add_library(${mbedtls_static_target} STATIC ${src_tls})
     set_target_properties(${mbedtls_static_target} PROPERTIES OUTPUT_NAME mbedtls)
     target_link_libraries(${mbedtls_static_target} ${libs} ${mbedx509_static_target})
-    target_include_directories(${mbedtls_static_target}
-        PUBLIC ${MBEDTLS_DIR}/include/)
-
-    install(TARGETS ${mbedtls_static_target} ${mbedx509_static_target} ${mbedcrypto_static_target}
-            DESTINATION ${LIB_INSTALL_DIR}
-            PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 endif(USE_STATIC_MBEDTLS_LIBRARY)
 
 if(USE_SHARED_MBEDTLS_LIBRARY)
-
     add_library(mbedcrypto SHARED ${src_crypto})
     set_target_properties(mbedcrypto PROPERTIES VERSION 2.23.0 SOVERSION 5)
     target_link_libraries(mbedcrypto ${libs})
-    target_include_directories(mbedcrypto
-        PUBLIC ${MBEDTLS_DIR}/include/
-        PUBLIC ${thirdparty_inc_public}
-        PRIVATE ${thirdparty_inc})
-    target_compile_definitions(mbedcrypto
-        PRIVATE ${thirdparty_def})
 
     add_library(mbedx509 SHARED ${src_x509})
     set_target_properties(mbedx509 PROPERTIES VERSION 2.23.0 SOVERSION 1)
     target_link_libraries(mbedx509 ${libs} mbedcrypto)
-    target_include_directories(mbedx509
-        PUBLIC ${MBEDTLS_DIR}/include/)
 
     add_library(mbedtls SHARED ${src_tls})
     set_target_properties(mbedtls PROPERTIES VERSION 2.23.0 SOVERSION 13)
     target_link_libraries(mbedtls ${libs} mbedx509)
-    target_include_directories(mbedtls
-        PUBLIC ${MBEDTLS_DIR}/include/)
-
-    install(TARGETS mbedtls mbedx509 mbedcrypto
-            DESTINATION ${LIB_INSTALL_DIR}
-            PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 endif(USE_SHARED_MBEDTLS_LIBRARY)
 
+foreach(target IN LISTS target_libraries)
+    # Include public header files from /include and other directories
+    # declared by /3rdparty/**/CMakeLists.txt. Include private header files
+    # from /library and others declared by /3rdparty/**/CMakeLists.txt.
+    # /library needs to be listed explicitly when building .c files outside
+    # of /library (which currently means: under /3rdparty).
+    target_include_directories(${target}
+        PUBLIC ${MBEDTLS_DIR}/include/
+        PUBLIC ${thirdparty_inc_public}
+        PRIVATE ${MBEDTLS_DIR}/library/
+        PRIVATE ${thirdparty_inc})
+    target_compile_definitions(${target}
+        PRIVATE ${thirdparty_def})
+    install(TARGETS ${target}
+            DESTINATION ${LIB_INSTALL_DIR}
+            PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ)
+endforeach(target)
+
 add_custom_target(lib DEPENDS mbedcrypto mbedx509 mbedtls)
 if(USE_STATIC_MBEDTLS_LIBRARY AND USE_SHARED_MBEDTLS_LIBRARY)
     add_dependencies(lib mbedcrypto_static mbedx509_static mbedtls_static)

library/Makefile

@@ -5,7 +5,11 @@ CFLAGS	?= -O2
 WARNING_CFLAGS ?=  -Wall -Wextra
 LDFLAGS ?=
 
-LOCAL_CFLAGS = $(WARNING_CFLAGS) -I../include -D_FILE_OFFSET_BITS=64
+# Include ../include for public headers and . for private headers.
+# Note that . needs to be included explicitly for the sake of library
+# files that are not in the /library directory (which currently means
+# under /3rdparty).
+LOCAL_CFLAGS = $(WARNING_CFLAGS) -I. -I../include -D_FILE_OFFSET_BITS=64
 LOCAL_LDFLAGS =
 
 ifdef DEBUG

library/aes.c

@@ -25,11 +25,7 @@
  *  http://csrc.nist.gov/publications/fips/fips197/fips-197.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_AES_C)
 

library/aesni.c

@@ -24,11 +24,7 @@
  * [CLMUL-WP] http://software.intel.com/en-us/articles/intel-carry-less-multiplication-instruction-and-its-usage-for-computing-the-gcm-mode/
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_AESNI_C)
 

library/arc4.c

@@ -24,11 +24,7 @@
  *  http://groups.google.com/group/sci.crypt/msg/10a300c9d21afca0
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ARC4_C)
 

library/aria.c

@@ -25,11 +25,7 @@
  * [2] https://tools.ietf.org/html/rfc5794
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ARIA_C)
 

library/asn1parse.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ASN1_PARSE_C)
 

library/asn1write.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ASN1_WRITE_C)
 

library/base64.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_BASE64_C)
 

library/bignum.c

@@ -35,11 +35,7 @@
  *
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_BIGNUM_C)
 

library/blowfish.c

@@ -25,11 +25,7 @@
  *
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_BLOWFISH_C)
 

library/camellia.c

@@ -25,11 +25,7 @@
  *  http://info.isl.ntt.co.jp/crypt/eng/camellia/dl/01espec.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CAMELLIA_C)
 

library/ccm.c

@@ -28,11 +28,7 @@
  * RFC 5116 "An Interface and Algorithms for Authenticated Encryption"
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CCM_C)
 

library/certs.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #include "mbedtls/certs.h"
 

library/chacha20.c

@@ -23,11 +23,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CHACHA20_C)
 

library/chachapoly.c

@@ -20,11 +20,7 @@
  *
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CHACHAPOLY_C)
 

library/cipher.c

@@ -23,11 +23,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CIPHER_C)
 

library/cipher_wrap.c

@@ -23,11 +23,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CIPHER_C)
 

library/cmac.c

@@ -40,11 +40,7 @@
  *
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CMAC_C)
 

library/ctr_drbg.c

@@ -24,11 +24,7 @@
  *  http://csrc.nist.gov/publications/nistpubs/800-90/SP800-90revised_March2007.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_CTR_DRBG_C)
 

library/debug.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_DEBUG_C)
 

library/des.c

@@ -25,11 +25,7 @@
  *  http://csrc.nist.gov/publications/fips/fips46-3/fips46-3.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_DES_C)
 

library/dhm.c

@@ -27,11 +27,7 @@
  *
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_DHM_C)
 

library/ecdh.c

@@ -26,11 +26,7 @@
  * RFC 4492
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ECDH_C)
 

library/ecdsa.c

@@ -25,11 +25,7 @@
  * SEC1 http://www.secg.org/index.php?action=secg,docs_secg
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ECDSA_C)
 

library/ecjpake.c

@@ -24,11 +24,7 @@
  * available to members of the Thread Group http://threadgroup.org/
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ECJPAKE_C)
 

library/ecp.c

@@ -41,11 +41,7 @@
  *     <http://eprint.iacr.org/2004/342.pdf>
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 /**
  * \brief Function level alternative implementation.

library/ecp_curves.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ECP_C)
 

library/entropy.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ENTROPY_C)
 

library/entropy_poll.c

@@ -24,11 +24,7 @@
 #define _GNU_SOURCE
 #endif
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #include <string.h>
 

library/error.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ERROR_STRERROR_DUMMY)
 #include <string.h>

library/gcm.c

@@ -29,11 +29,7 @@
  * [MGV] 4.1, pp. 12-13, to enhance speed without using too much memory.
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_GCM_C)
 

library/havege.c

@@ -26,11 +26,7 @@
  *  Contact: seznec(at)irisa_dot_fr - orocheco(at)irisa_dot_fr
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_HAVEGE_C)
 

library/hkdf.c

@@ -18,11 +18,7 @@
  *
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_HKDF_C)
 

library/hmac_drbg.c

@@ -25,11 +25,7 @@
  *  References below are based on rev. 1 (January 2012).
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_HMAC_DRBG_C)
 

library/md.c

@@ -23,11 +23,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_MD_C)
 

library/md2.c

@@ -25,11 +25,7 @@
  *  http://www.ietf.org/rfc/rfc1319.txt
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_MD2_C)
 

library/md4.c

@@ -25,11 +25,7 @@
  *  http://www.ietf.org/rfc/rfc1320.txt
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_MD4_C)
 

library/md5.c

@@ -24,11 +24,7 @@
  *  http://www.ietf.org/rfc/rfc1321.txt
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_MD5_C)
 

library/memory_buffer_alloc.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
 #include "mbedtls/memory_buffer_alloc.h"

library/net_sockets.c

@@ -25,11 +25,7 @@
 #define _POSIX_C_SOURCE 200112L
 #define _XOPEN_SOURCE 600 /* sockaddr_storage */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_NET_C)
 

library/nist_kw.c

@@ -29,11 +29,7 @@
  * the wrapping and unwrapping operation than the definition in NIST SP 800-38F.
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_NIST_KW_C)
 

library/oid.c

@@ -21,11 +21,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_OID_C)
 

library/padlock.c

@@ -25,11 +25,7 @@
  *  programming_guide.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PADLOCK_C)
 

library/pem.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PEM_PARSE_C) || defined(MBEDTLS_PEM_WRITE_C)
 

library/pk.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PK_C)
 #include "mbedtls/pk.h"

library/pk_wrap.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PK_C)
 #include "mbedtls/pk_internal.h"

library/pkcs12.c

@@ -25,11 +25,7 @@
  *  ftp://ftp.rsasecurity.com/pub/pkcs/pkcs-12/pkcs-12v1-1.asn
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PKCS12_C)
 

library/pkcs5.c

@@ -29,11 +29,7 @@
  * http://tools.ietf.org/html/rfc6070 (Test vectors)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PKCS5_C)
 

library/pkparse.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PK_PARSE_C)
 

library/pkwrite.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PK_WRITE_C)
 

library/platform.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PLATFORM_C)
 

library/platform_util.c

@@ -28,11 +28,7 @@
 #define _POSIX_C_SOURCE 200112L
 #endif
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #include "mbedtls/platform_util.h"
 #include "mbedtls/platform.h"

library/poly1305.c

@@ -20,11 +20,7 @@
  *
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_POLY1305_C)
 

library/psa_crypto.c

@@ -20,11 +20,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PSA_CRYPTO_C)
 

library/psa_crypto_se.c

@@ -20,11 +20,7 @@
  *  This file is part of Mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PSA_CRYPTO_SE_C)
 

library/psa_crypto_slot_management.c

@@ -20,11 +20,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_PSA_CRYPTO_C)
 

library/ripemd160.c

@@ -25,11 +25,7 @@
  *  http://ehash.iaik.tugraz.at/wiki/RIPEMD-160
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_RIPEMD160_C)
 

library/rsa.c

@@ -37,11 +37,7 @@
  *
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_RSA_C)
 

library/rsa_internal.c

@@ -20,11 +20,7 @@
  *
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_RSA_C)
 

library/sha1.c

@@ -24,11 +24,7 @@
  *  http://www.itl.nist.gov/fipspubs/fip180-1.htm
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SHA1_C)
 

library/sha256.c

@@ -24,11 +24,7 @@
  *  http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SHA256_C)
 

library/sha512.c

@@ -24,11 +24,7 @@
  *  http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SHA512_C)
 

library/ssl_cache.c

@@ -23,11 +23,7 @@
  * to store and retrieve the session information.
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_CACHE_C)
 

library/ssl_ciphersuites.c

@@ -21,11 +21,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_TLS_C)
 

library/ssl_cli.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_CLI_C)
 

library/ssl_cookie.c

@@ -23,11 +23,7 @@
  * to store and retrieve the session information.
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_COOKIE_C)
 

library/ssl_msg.c

@@ -28,11 +28,7 @@
  *  http://www.ietf.org/rfc/rfc4346.txt
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_TLS_C)
 

library/ssl_srv.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_SRV_C)
 

library/ssl_ticket.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_TICKET_C)
 

library/ssl_tls.c

@@ -27,11 +27,7 @@
  *  http://www.ietf.org/rfc/rfc4346.txt
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SSL_TLS_C)
 

library/threading.c

@@ -27,11 +27,7 @@
 #define _POSIX_C_SOURCE 200112L
 #endif
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_THREADING_C)
 

library/timing.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_PLATFORM_C)
 #include "mbedtls/platform.h"

library/version.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_VERSION_C)
 

library/version_features.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_VERSION_C)
 

library/x509.c

@@ -29,11 +29,7 @@
  *  http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_X509_USE_C)
 

library/x509_create.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_X509_CREATE_C)
 

library/x509_crl.c

@@ -29,11 +29,7 @@
  *  http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_X509_CRL_PARSE_C)
 

library/x509_crt.c

@@ -31,11 +31,7 @@
  *  [SIRO] https://cabforum.org/wp-content/uploads/Chunghwatelecom201503cabforumV4.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_X509_CRT_PARSE_C)
 

library/x509_csr.c

@@ -29,11 +29,7 @@
  *  http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_X509_CSR_PARSE_C)
 

library/x509write_crt.c

@@ -25,11 +25,7 @@
  * - attributes: PKCS#9 v2.0 aka RFC 2985
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_X509_CRT_WRITE_C)
 

library/x509write_csr.c

@@ -24,11 +24,7 @@
  * - attributes: PKCS#9 v2.0 aka RFC 2985
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_X509_CSR_WRITE_C)
 

library/xtea.c

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_XTEA_C)
 

scripts/data_files/error.fmt

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_ERROR_STRERROR_DUMMY)
 #include <string.h>

scripts/data_files/version_features.fmt

@@ -19,11 +19,7 @@
  *  This file is part of mbed TLS (https://tls.mbed.org)
  */
 
-#if !defined(MBEDTLS_CONFIG_FILE)
+#include "common.h"
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
 
 #if defined(MBEDTLS_VERSION_C)
 

scripts/generate_visualc_files.pl

@@ -64,6 +64,15 @@ my @include_directories = qw(
 );
 my $include_directories = join(';', map {"../../$_"} @include_directories);
 
+# Directories to add to the include path when building the library, but not
+# when building tests or applications.
+my @library_include_directories = qw(
+    library
+);
+my $library_include_directories =
+  join(';', map {"../../$_"} (@library_include_directories,
+                              @include_directories));
+
 my @excluded_files = qw(
     3rdparty/everest/library/Hacl_Curve25519.c
 );
@@ -202,7 +211,7 @@ sub gen_main_file {
     my $out = slurp_file( $main_tpl );
     $out =~ s/SOURCE_ENTRIES\r\n/$source_entries/m;
     $out =~ s/HEADER_ENTRIES\r\n/$header_entries/m;
-    $out =~ s/INCLUDE_DIRECTORIES\r\n/$include_directories/g;
+    $out =~ s/INCLUDE_DIRECTORIES\r\n/$library_include_directories/g;
 
     content_to_file( $out, $main_out );
 }

tests/CMakeLists.txt

@@ -48,6 +48,10 @@ function(add_test_suite suite_name)
 
     add_executable(test_suite_${data_name} test_suite_${data_name}.c $<TARGET_OBJECTS:mbedtls_test>)
     target_link_libraries(test_suite_${data_name} ${libs})
+    # Include test-specific header files from ./include and private header
+    # files (used by some invasive tests) from ../library. Public header
+    # files are automatically included because the library targets declare
+    # them as PUBLIC.
     target_include_directories(test_suite_${data_name}
         PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
         PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../library)

tests/Makefile

@@ -6,6 +6,9 @@ CFLAGS	?= -O2
 WARNING_CFLAGS ?= -Wall -Wextra
 LDFLAGS ?=
 
+# Include public header files from ../include, test-specific header files
+# from ./include, and private header files (used by some invasive tests)
+# from ../library.
 LOCAL_CFLAGS = $(WARNING_CFLAGS) -I./include -I../include -I../library -D_FILE_OFFSET_BITS=64
 LOCAL_LDFLAGS = -L../library			\
 		-lmbedtls$(SHARED_SUFFIX)	\

tests/scripts/all.sh

@@ -680,7 +680,7 @@ component_check_doxy_blocks () {
 
 component_check_files () {
     msg "Check: file sanity checks (permissions, encodings)" # < 1s
-    record_status tests/scripts/check-files.py
+    record_status tests/scripts/check_files.py
 }
 
 component_check_changelog () {
@@ -707,7 +707,7 @@ component_check_test_cases () {
     else
         opt=''
     fi
-    record_status tests/scripts/check-test-cases.py $opt
+    record_status tests/scripts/check_test_cases.py $opt
     unset opt
 }
 

tests/scripts/analyze_outcomes.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+"""Analyze the test outcomes from a full CI run.
+
+This script can also run on outcomes from a partial run, but the results are
+less likely to be useful.
+"""
+
+import argparse
+import re
+import sys
+import traceback
+
+import check_test_cases
+
+class Results:
+    """Process analysis results."""
+
+    def __init__(self):
+        self.error_count = 0
+        self.warning_count = 0
+
+    @staticmethod
+    def log(fmt, *args, **kwargs):
+        sys.stderr.write((fmt + '\n').format(*args, **kwargs))
+
+    def error(self, fmt, *args, **kwargs):
+        self.log('Error: ' + fmt, *args, **kwargs)
+        self.error_count += 1
+
+    def warning(self, fmt, *args, **kwargs):
+        self.log('Warning: ' + fmt, *args, **kwargs)
+        self.warning_count += 1
+
+class TestCaseOutcomes:
+    """The outcomes of one test case across many configurations."""
+    # pylint: disable=too-few-public-methods
+
+    def __init__(self):
+        # Collect a list of witnesses of the test case succeeding or failing.
+        # Currently we don't do anything with witnesses except count them.
+        # The format of a witness is determined by the read_outcome_file
+        # function; it's the platform and configuration joined by ';'.
+        self.successes = []
+        self.failures = []
+
+    def hits(self):
+        """Return the number of times a test case has been run.
+
+        This includes passes and failures, but not skips.
+        """
+        return len(self.successes) + len(self.failures)
+
+class TestDescriptions(check_test_cases.TestDescriptionExplorer):
+    """Collect the available test cases."""
+
+    def __init__(self):
+        super().__init__()
+        self.descriptions = set()
+
+    def process_test_case(self, _per_file_state,
+                          file_name, _line_number, description):
+        """Record an available test case."""
+        base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
+        key = ';'.join([base_name, description.decode('utf-8')])
+        self.descriptions.add(key)
+
+def collect_available_test_cases():
+    """Collect the available test cases."""
+    explorer = TestDescriptions()
+    explorer.walk_all()
+    return sorted(explorer.descriptions)
+
+def analyze_coverage(results, outcomes):
+    """Check that all available test cases are executed at least once."""
+    available = collect_available_test_cases()
+    for key in available:
+        hits = outcomes[key].hits() if key in outcomes else 0
+        if hits == 0:
+            # Make this a warning, not an error, as long as we haven't
+            # fixed this branch to have full coverage of test cases.
+            results.warning('Test case not executed: {}', key)
+
+def analyze_outcomes(outcomes):
+    """Run all analyses on the given outcome collection."""
+    results = Results()
+    analyze_coverage(results, outcomes)
+    return results
+
+def read_outcome_file(outcome_file):
+    """Parse an outcome file and return an outcome collection.
+
+An outcome collection is a dictionary mapping keys to TestCaseOutcomes objects.
+The keys are the test suite name and the test case description, separated
+by a semicolon.
+"""
+    outcomes = {}
+    with open(outcome_file, 'r', encoding='utf-8') as input_file:
+        for line in input_file:
+            (platform, config, suite, case, result, _cause) = line.split(';')
+            key = ';'.join([suite, case])
+            setup = ';'.join([platform, config])
+            if key not in outcomes:
+                outcomes[key] = TestCaseOutcomes()
+            if result == 'PASS':
+                outcomes[key].successes.append(setup)
+            elif result == 'FAIL':
+                outcomes[key].failures.append(setup)
+    return outcomes
+
+def analyze_outcome_file(outcome_file):
+    """Analyze the given outcome file."""
+    outcomes = read_outcome_file(outcome_file)
+    return analyze_outcomes(outcomes)
+
+def main():
+    try:
+        parser = argparse.ArgumentParser(description=__doc__)
+        parser.add_argument('outcomes', metavar='OUTCOMES.CSV',
+                            help='Outcome file to analyze')
+        options = parser.parse_args()
+        results = analyze_outcome_file(options.outcomes)
+        if results.error_count > 0:
+            sys.exit(1)
+    except Exception: # pylint: disable=broad-except
+        # Print the backtrace and exit explicitly with our chosen status.
+        traceback.print_exc()
+        sys.exit(120)
+
+if __name__ == '__main__':
+    main()

tests/scripts/basic-in-docker.sh

@@ -4,8 +4,10 @@
 #
 # Purpose
 # -------
-# This runs a rough equivalent of the travis.yml in a Docker container.
+# This runs sanity checks and library tests in a Docker container. The tests
-# The tests are run for both clang and gcc.
+# are run for both clang and gcc. The testing includes a full test run
+# in the default configuration, partial test runs in the reference
+# configurations, and some dependency tests.
 #
 # Notes for users
 # ---------------
@@ -30,12 +32,7 @@
 
 source tests/scripts/docker_env.sh
 
-run_in_docker tests/scripts/recursion.pl library/*.c
+run_in_docker tests/scripts/all.sh 'check_*'
-run_in_docker tests/scripts/check-generated-files.sh
-run_in_docker tests/scripts/check-doxy-blocks.pl
-run_in_docker tests/scripts/check-names.sh
-run_in_docker tests/scripts/check-files.py
-run_in_docker tests/scripts/doxygen.sh
 
 for compiler in clang gcc; do
     run_in_docker -e CC=${compiler} cmake -D CMAKE_BUILD_TYPE:String="Check" .

tests/scripts/check-test-cases.py

@@ -1,136 +0,0 @@
-#!/usr/bin/env python3
-
-"""Sanity checks for test data.
-"""
-
-# Copyright (C) 2019, Arm Limited, All Rights Reserved
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This file is part of Mbed TLS (https://tls.mbed.org)
-
-import argparse
-import glob
-import os
-import re
-import sys
-
-class Results:
-    """Store file and line information about errors or warnings in test suites."""
-
-    def __init__(self, options):
-        self.errors = 0
-        self.warnings = 0
-        self.ignore_warnings = options.quiet
-
-    def error(self, file_name, line_number, fmt, *args):
-        sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
-                         format(file_name, line_number, *args))
-        self.errors += 1
-
-    def warning(self, file_name, line_number, fmt, *args):
-        if not self.ignore_warnings:
-            sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
-                             .format(file_name, line_number, *args))
-            self.warnings += 1
-
-def collect_test_directories():
-    """Get the relative path for the TLS and Crypto test directories."""
-    if os.path.isdir('tests'):
-        tests_dir = 'tests'
-    elif os.path.isdir('suites'):
-        tests_dir = '.'
-    elif os.path.isdir('../suites'):
-        tests_dir = '..'
-    directories = [tests_dir]
-    return directories
-
-def check_description(results, seen, file_name, line_number, description):
-    """Check test case descriptions for errors."""
-    if description in seen:
-        results.error(file_name, line_number,
-                      'Duplicate description (also line {})',
-                      seen[description])
-        return
-    if re.search(br'[\t;]', description):
-        results.error(file_name, line_number,
-                      'Forbidden character \'{}\' in description',
-                      re.search(br'[\t;]', description).group(0).decode('ascii'))
-    if re.search(br'[^ -~]', description):
-        results.error(file_name, line_number,
-                      'Non-ASCII character in description')
-    if len(description) > 66:
-        results.warning(file_name, line_number,
-                        'Test description too long ({} > 66)',
-                        len(description))
-    seen[description] = line_number
-
-def check_test_suite(results, data_file_name):
-    """Check the test cases in the given unit test data file."""
-    in_paragraph = False
-    descriptions = {}
-    with open(data_file_name, 'rb') as data_file:
-        for line_number, line in enumerate(data_file, 1):
-            line = line.rstrip(b'\r\n')
-            if not line:
-                in_paragraph = False
-                continue
-            if line.startswith(b'#'):
-                continue
-            if not in_paragraph:
-                # This is a test case description line.
-                check_description(results, descriptions,
-                                  data_file_name, line_number, line)
-            in_paragraph = True
-
-def check_ssl_opt_sh(results, file_name):
-    """Check the test cases in ssl-opt.sh or a file with a similar format."""
-    descriptions = {}
-    with open(file_name, 'rb') as file_contents:
-        for line_number, line in enumerate(file_contents, 1):
-            # Assume that all run_test calls have the same simple form
-            # with the test description entirely on the same line as the
-            # function name.
-            m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
-            if not m:
-                continue
-            description = m.group(1)
-            check_description(results, descriptions,
-                              file_name, line_number, description)
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--quiet', '-q',
-                        action='store_true',
-                        help='Hide warnings')
-    parser.add_argument('--verbose', '-v',
-                        action='store_false', dest='quiet',
-                        help='Show warnings (default: on; undoes --quiet)')
-    options = parser.parse_args()
-    test_directories = collect_test_directories()
-    results = Results(options)
-    for directory in test_directories:
-        for data_file_name in glob.glob(os.path.join(directory, 'suites',
-                                                     '*.data')):
-            check_test_suite(results, data_file_name)
-        ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
-        if os.path.exists(ssl_opt_sh):
-            check_ssl_opt_sh(results, ssl_opt_sh)
-    if (results.warnings or results.errors) and not options.quiet:
-        sys.stderr.write('{}: {} errors, {} warnings\n'
-                         .format(sys.argv[0], results.errors, results.warnings))
-    sys.exit(1 if results.errors else 0)
-
-if __name__ == '__main__':
-    main()