Merge pull request #7165 from yanrayw/7094-collect-compatsh-test-cases

check_test_cases.py: support to collect test cases for compat.sh

tests/compat.sh

@@ -126,10 +126,41 @@ print_usage() {
     printf "            \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
     printf "  -M|--memcheck\tCheck memory leaks and errors.\n"
     printf "  -v|--verbose\tSet verbose output.\n"
+    printf "     --list-test-case\tList all potential test cases (No Execution)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                   \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
 }
 
+# print_test_case <CLIENT> <SERVER> <STANDARD_CIPHER_SUITE>
+print_test_case() {
+    for i in $3; do
+        uniform_title $1 $2 $i
+        echo $TITLE
+    done
+}
+
+# list_test_case lists all potential test cases in compat.sh without execution
+list_test_case() {
+    reset_ciphersuites
+    for TYPE in $TYPES; do
+        add_common_ciphersuites
+        add_openssl_ciphersuites
+        add_gnutls_ciphersuites
+        add_mbedtls_ciphersuites
+    done
+
+    for VERIFY in $VERIFIES; do
+        VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]')
+        for MODE in $MODES; do
+            print_test_case m O "$O_CIPHERS"
+            print_test_case O m "$O_CIPHERS"
+            print_test_case m G "$G_CIPHERS"
+            print_test_case G m "$G_CIPHERS"
+            print_test_case m m "$M_CIPHERS"
+        done
+    done
+}
+
 get_options() {
     while [ $# -gt 0 ]; do
         case "$1" in
@@ -157,6 +188,12 @@ get_options() {
             -M|--memcheck)
                 MEMCHECK=1
                 ;;
+            # Please check scripts/check_test_cases.py correspondingly
+            # if you have to modify option, --list-test-case
+            --list-test-case)
+                list_test_case
+                exit $?
+                ;;
             --outcome-file)
                 shift; MBEDTLS_TEST_OUTCOME_FILE=$1
                 ;;
@@ -826,6 +863,14 @@ wait_client_done() {
     echo "EXIT: $EXIT" >> $CLI_OUT
 }
 
+# uniform_title <CLIENT> <SERVER> <STANDARD_CIPHER_SUITE>
+# $TITLE is considered as test case description for both --list-test-case and
+# MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of
+# each test case description.
+uniform_title() {
+    TITLE="$1->$2 $MODE,$VERIF $3"
+}
+
 # record_outcome <outcome> [<failure-reason>]
 record_outcome() {
     echo "$1"
@@ -863,8 +908,7 @@ report_fail() {
 run_client() {
     # announce what we're going to do
     TESTS=$(( $TESTS + 1 ))
-    TITLE="${1%"${1#?}"}->${SERVER_NAME%"${SERVER_NAME#?}"}"
-    TITLE="$TITLE $MODE,$VERIF $2"
+    uniform_title "${1%"${1#?}"}" "${SERVER_NAME%"${SERVER_NAME#?}"}" $2
     DOTS72="........................................................................"
     printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72"
 

tests/scripts/check_test_cases.py

@@ -25,6 +25,7 @@ import argparse
 import glob
 import os
 import re
+import subprocess
 import sys
 
 class Results:
@@ -111,6 +112,19 @@ state may override this method.
                 self.process_test_case(descriptions,
                                        file_name, line_number, description)
 
+    def walk_compat_sh(self, file_name):
+        """Iterate over the test cases compat.sh with a similar format."""
+        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+        compat_cmd = ['sh', file_name, '--list-test-case']
+        compat_output = subprocess.check_output(compat_cmd)
+        # Assume compat.sh is responsible for printing identical format of
+        # test case description between --list-test-case and its OUTCOME.CSV
+        description = compat_output.strip().split(b'\n')
+        # idx indicates the number of test case since there is no line number
+        # in `compat.sh` for each test case.
+        for idx, descrip in enumerate(description):
+            self.process_test_case(descriptions, file_name, idx, descrip)
+
     @staticmethod
     def collect_test_directories():
         """Get the relative path for the TLS and Crypto test directories."""
@@ -136,6 +150,9 @@ state may override this method.
             for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases',
                                                             '*.sh')):
                 self.walk_ssl_opt_sh(ssl_opt_file_name)
+            compat_sh = os.path.join(directory, 'compat.sh')
+            if os.path.exists(compat_sh):
+                self.walk_compat_sh(compat_sh)
 
 class TestDescriptions(TestDescriptionExplorer):
     """Collect the available test cases."""