Merge pull request #5393 from gilles-peskine-arm/opt-testcases-outcomes-fix

Fix test suite name reporting of opt-testcases/tls13-compat.sh

tests/scripts/analyze_outcomes.py

@@ -7,7 +7,6 @@ less likely to be useful.
 """
 
 import argparse
-import re
 import sys
 import traceback
 
@@ -51,29 +50,9 @@ class TestCaseOutcomes:
         """
         return len(self.successes) + len(self.failures)
 
-class TestDescriptions(check_test_cases.TestDescriptionExplorer):
-    """Collect the available test cases."""
-
-    def __init__(self):
-        super().__init__()
-        self.descriptions = set()
-
-    def process_test_case(self, _per_file_state,
-                          file_name, _line_number, description):
-        """Record an available test case."""
-        base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
-        key = ';'.join([base_name, description.decode('utf-8')])
-        self.descriptions.add(key)
-
-def collect_available_test_cases():
-    """Collect the available test cases."""
-    explorer = TestDescriptions()
-    explorer.walk_all()
-    return sorted(explorer.descriptions)
-
 def analyze_coverage(results, outcomes):
     """Check that all available test cases are executed at least once."""
-    available = collect_available_test_cases()
+    available = check_test_cases.collect_available_test_cases()
     for key in available:
         hits = outcomes[key].hits() if key in outcomes else 0
         if hits == 0:

tests/scripts/check_test_cases.py

@@ -137,6 +137,26 @@ state may override this method.
                                                             '*.sh')):
                 self.walk_ssl_opt_sh(ssl_opt_file_name)
 
+class TestDescriptions(TestDescriptionExplorer):
+    """Collect the available test cases."""
+
+    def __init__(self):
+        super().__init__()
+        self.descriptions = set()
+
+    def process_test_case(self, _per_file_state,
+                          file_name, _line_number, description):
+        """Record an available test case."""
+        base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
+        key = ';'.join([base_name, description.decode('utf-8')])
+        self.descriptions.add(key)
+
+def collect_available_test_cases():
+    """Collect the available test cases."""
+    explorer = TestDescriptions()
+    explorer.walk_all()
+    return sorted(explorer.descriptions)
+
 class DescriptionChecker(TestDescriptionExplorer):
     """Check all test case descriptions.
 
@@ -176,6 +196,9 @@ class DescriptionChecker(TestDescriptionExplorer):
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--list-all',
+                        action='store_true',
+                        help='List all test cases, without doing checks')
     parser.add_argument('--quiet', '-q',
                         action='store_true',
                         help='Hide warnings')
@@ -183,6 +206,10 @@ def main():
                         action='store_false', dest='quiet',
                         help='Show warnings (default: on; undoes --quiet)')
     options = parser.parse_args()
+    if options.list_all:
+        descriptions = collect_available_test_cases()
+        sys.stdout.write('\n'.join(descriptions + ['']))
+        return
     results = Results(options)
     checker = DescriptionChecker(results)
     checker.walk_all()

tests/ssl-opt.sh

@@ -543,16 +543,18 @@ print_name() {
 
 # record_outcome <outcome> [<failure-reason>]
 # The test name must be in $NAME.
+# Use $TEST_SUITE_NAME as the test suite name if set.
 record_outcome() {
     echo "$1"
     if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
         printf '%s;%s;%s;%s;%s;%s\n' \
                "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
-               "ssl-opt" "$NAME" \
+               "${TEST_SUITE_NAME:-ssl-opt}" "$NAME" \
                "$1" "${2-}" \
                >>"$MBEDTLS_TEST_OUTCOME_FILE"
     fi
 }
+unset TEST_SUITE_NAME
 
 # True if the presence of the given pattern in a log definitely indicates
 # that the test has failed. False if the presence is inconclusive.
@@ -9281,10 +9283,13 @@ run_test    "TLS 1.3: HelloRetryRequest check, ciphersuite TLS_AES_256_GCM_SHA38
             -c "tls13 client state: MBEDTLS_SSL_CLIENT_HELLO" \
             -c "HTTP/1.0 200 OK"
 
-for i in $(ls opt-testcases/*.sh)
+for i in opt-testcases/*.sh
 do
-    . $i
+    TEST_SUITE_NAME=${i##*/}
+    TEST_SUITE_NAME=${TEST_SUITE_NAME%.*}
+    . "$i"
 done
+unset TEST_SUITE_NAME
 
 requires_openssl_tls1_3
 requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3