Valgrind for constant flow: skip non-CF test suites

When testing under Valgrind for constant flow, skip test suites that don't
have any constant-flow annotations, since the testing wouldn't do anything
more that testing with ordinary Valgrind (component_test_valgrind and
component_test_valgrind_psa). This is a significant time saving since
testing with Valgrind is very slow.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>

tests/include/test/constant_flow.h

@@ -46,6 +46,12 @@
  * This file contains two implementations: one based on MemorySanitizer, the
  * other on valgrind's memcheck. If none of them is enabled, dummy macros that
  * do nothing are defined for convenience.
+ *
+ * \note #TEST_CF_SECRET must be called directly from within a .function file,
+ *       not indirectly via a macro defined under tests/include or a function
+ *       under tests/src. This is because we only run Valgrind for constant
+ *       flow on test suites that have greppable annotations inside them (see
+ *       `skip_suites_without_constant_flow` in `tests/scripts/all.sh`).
  */
 
 #if defined(MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN)

tests/scripts/all.sh

@@ -1591,6 +1591,17 @@ component_test_full_cmake_clang () {
     env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA'
 }
 
+skip_suites_without_constant_flow () {
+    # Skip the test suites that don't have any constant-flow annotations.
+    # This will need to be adjusted if we ever start declaring things as
+    # secret from macros or functions inside tests/include or tests/src.
+    SKIP_TEST_SUITES=$(
+        git -C tests/suites grep -L TEST_CF_ 'test_suite_*.function' |
+            sed 's/test_suite_//; s/\.function$//' |
+            tr '\n' ,)
+    export SKIP_TEST_SUITES
+}
+
 component_test_memsan_constant_flow () {
     # This tests both (1) accesses to undefined memory, and (2) branches or
     # memory access depending on secret values. To distinguish between those:
@@ -1642,12 +1653,13 @@ component_test_valgrind_constant_flow () {
     scripts/config.py full
     scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND
     scripts/config.py unset MBEDTLS_USE_PSA_CRYPTO
+    skip_suites_without_constant_flow
     cmake -D CMAKE_BUILD_TYPE:String=Release .
     make
 
     # this only shows a summary of the results (how many of each type)
     # details are left in Testing/<date>/DynamicAnalysis.xml
-    msg "test: main suites (full minus MBEDTLS_USE_PSA_CRYPTO, valgrind + constant flow)"
+    msg "test: some suites (full minus MBEDTLS_USE_PSA_CRYPTO, valgrind + constant flow)"
     make memcheck
 }
 
@@ -1664,12 +1676,13 @@ component_test_valgrind_constant_flow_psa () {
     msg "build: cmake release GCC, full config with constant flow testing"
     scripts/config.py full
     scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND
+    skip_suites_without_constant_flow
     cmake -D CMAKE_BUILD_TYPE:String=Release .
     make
 
     # this only shows a summary of the results (how many of each type)
     # details are left in Testing/<date>/DynamicAnalysis.xml
-    msg "test: main suites (valgrind + constant flow)"
+    msg "test: some suites (valgrind + constant flow)"
     make memcheck
 }