diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 706421f6c..80b6459cd 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -95,9 +95,21 @@ def analyze_coverage(results, outcomes, allow_list, full_coverage): else: results.warning('Allow listed test case was executed: {}', key) +def name_matches_pattern(name, str_or_re): + """Check if name matches a pattern, that may be a string or regex. + - If the pattern is a string, name must be equal to match. + - If the pattern is a regex, name must fully match. + """ + # The CI's python is too old for re.Pattern + #if isinstance(str_or_re, re.Pattern): + if not isinstance(str_or_re, str): + return str_or_re.fullmatch(name) + else: + return str_or_re == name + def analyze_driver_vs_reference(results: Results, outcomes, component_ref, component_driver, - ignored_suites, ignored_test=None): + ignored_suites, ignored_tests=None): """Check that all tests executed in the reference component are also executed in the corresponding driver component. Skip: @@ -105,22 +117,25 @@ def analyze_driver_vs_reference(results: Results, outcomes, - only some specific test inside a test suite, for which the corresponding output string is provided """ - available = check_test_cases.collect_available_test_cases() - - for key in available: - # Continue if test was not executed by any component - hits = outcomes[key].hits() if key in outcomes else 0 - if hits == 0: - continue - # Skip ignored test suites - full_test_suite = key.split(';')[0] # retrieve full test suite name - test_string = key.split(';')[1] # retrieve the text string of this test + seen_reference_passing = False + for key in outcomes: + # key is like "test_suite_foo.bar;Description of test case" + (full_test_suite, test_string) = key.split(';') test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name + + # Immediately skip fully-ignored test suites if test_suite in ignored_suites or full_test_suite in ignored_suites: continue - if ((full_test_suite in ignored_test) and - (test_string in ignored_test[full_test_suite])): - continue + + # For ignored test cases inside test suites, just remember and: + # don't issue an error if they're skipped with drivers, + # but issue an error if they're not (means we have a bad entry). + ignored = False + if full_test_suite in ignored_tests: + for str_or_re in ignored_tests[test_suite]: + if name_matches_pattern(test_string, str_or_re): + ignored = True + # Search for tests that run in reference component and not in driver component driver_test_passed = False reference_test_passed = False @@ -129,8 +144,14 @@ def analyze_driver_vs_reference(results: Results, outcomes, driver_test_passed = True if component_ref in entry: reference_test_passed = True - if(reference_test_passed and not driver_test_passed): - results.error("Did not pass with driver: {}", key) + seen_reference_passing = True + if reference_test_passed and not driver_test_passed and not ignored: + results.error("PASS -> SKIP/FAIL: {}", key) + if ignored and driver_test_passed: + results.error("uselessly ignored: {}", key) + + if not seen_reference_passing: + results.error("no passing test in reference component: bad outcome file?") def analyze_outcomes(results: Results, outcomes, args): """Run all analyses on the given outcome collection.""" @@ -232,15 +253,7 @@ KNOWN_TASKS = { # PEM decryption is not supported so far. # The rest of PEM (write, unencrypted read) works though. 'test_suite_pem': [ - 'PEM read (AES-128-CBC + invalid iv)' - 'PEM read (DES-CBC + invalid iv)', - 'PEM read (DES-EDE3-CBC + invalid iv)', - 'PEM read (malformed PEM AES-128-CBC)', - 'PEM read (malformed PEM DES-CBC)', - 'PEM read (malformed PEM DES-EDE3-CBC)', - 'PEM read (unknown encryption algorithm)', - 'PEM read (AES-128-CBC + invalid iv)', - 'PEM read (DES-CBC + invalid iv)', + re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), ], # Following tests depend on AES_C/DES_C but are not about # them really, just need to know some error code is there. @@ -255,258 +268,21 @@ KNOWN_TASKS = { # The en/decryption part of PKCS#12 is not supported so far. # The rest of PKCS#12 (key derivation) works though. 'test_suite_pkcs12': [ - 'PBE Decrypt, (Invalid padding & PKCS7 padding enabled)', - 'PBE Decrypt, pad = 7 (OK)', - 'PBE Decrypt, pad = 8 (Invalid output size)', - 'PBE Decrypt, pad = 8 (OK)', - 'PBE Encrypt, pad = 7 (OK)', - 'PBE Encrypt, pad = 8 (Invalid output size)', - 'PBE Encrypt, pad = 8 (OK)', + re.compile(r'PBE Encrypt, .*'), + re.compile(r'PBE Decrypt, .*'), ], # The en/decryption part of PKCS#5 is not supported so far. # The rest of PKCS#5 (PBKDF2) works though. 'test_suite_pkcs5': [ - 'PBES2 Decrypt (Invalid output size)', - 'PBES2 Decrypt (Invalid padding & PKCS7 padding enabled)', - 'PBES2 Decrypt (KDF != PBKDF2)', - 'PBES2 Decrypt (OK)', - 'PBES2 Decrypt (OK, PBKDF2 params explicit keylen)', - 'PBES2 Decrypt (OK, PBKDF2 params explicit prf_alg)', - 'PBES2 Decrypt (bad KDF AlgId: not a sequence)', - 'PBES2 Decrypt (bad KDF AlgId: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params explicit keylen: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params iter: not an int)', - 'PBES2 Decrypt (bad PBKDF2 params iter: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params salt: not an octet string)', - 'PBES2 Decrypt (bad PBKDF2 params salt: overlong)', - 'PBES2 Decrypt (bad PBKDF2 params: not a sequence)', - 'PBES2 Decrypt (bad PBKDF2 params: overlong)', - 'PBES2 Decrypt (bad enc_scheme_alg params: len != iv_len)', - 'PBES2 Decrypt (bad enc_scheme_alg params: not an octet string)', - 'PBES2 Decrypt (bad enc_scheme_alg params: overlong)', - 'PBES2 Decrypt (bad enc_scheme_alg: not a sequence)', - 'PBES2 Decrypt (bad enc_scheme_alg: overlong)', - 'PBES2 Decrypt (bad enc_scheme_alg: unknown oid)', - 'PBES2 Decrypt (bad iter value)', - 'PBES2 Decrypt (bad params tag)', - 'PBES2 Decrypt (bad password)', - 'PBES2 Decrypt (bad, PBKDF2 params explicit prf_alg != HMAC-SHA*)', - 'PBES2 Decrypt (bad, PBKDF2 params explicit prf_alg not a sequence)', - 'PBES2 Decrypt (bad, PBKDF2 params explicit prf_alg overlong)', - 'PBES2 Decrypt (bad, PBKDF2 params extra data)', - 'PBES2 Encrypt, pad=6 (OK)', - 'PBES2 Encrypt, pad=8 (Invalid output size)', - 'PBES2 Encrypt, pad=8 (OK)', + re.compile(r'PBES2 Encrypt, .*'), + re.compile(r'PBES2 Decrypt .*'), ], # Encrypted keys are not supported so far. # pylint: disable=line-too-long 'test_suite_pkparse': [ 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', - 'Parse RSA Key #20 (PKCS#8 encrypted SHA1-3DES)', - 'Parse RSA Key #20.1 (PKCS#8 encrypted SHA1-3DES, wrong PW)', - 'Parse RSA Key #20.2 (PKCS#8 encrypted SHA1-3DES, no PW)', - 'Parse RSA Key #21 (PKCS#8 encrypted SHA1-3DES, 2048-bit)', - 'Parse RSA Key #21.1 (PKCS#8 encrypted SHA1-3DES, 2048-bit, wrong PW)', - 'Parse RSA Key #21.2 (PKCS#8 encrypted SHA1-3DES, 2048-bit, no PW)', - 'Parse RSA Key #22 (PKCS#8 encrypted SHA1-3DES, 4096-bit)', - 'Parse RSA Key #22.1 (PKCS#8 encrypted SHA1-3DES, 4096-bit, wrong PW)', - 'Parse RSA Key #22.2 (PKCS#8 encrypted SHA1-3DES, 4096-bit, no PW)', - 'Parse RSA Key #23 (PKCS#8 encrypted SHA1-3DES DER)', - 'Parse RSA Key #24 (PKCS#8 encrypted SHA1-3DES DER, 2048-bit)', - 'Parse RSA Key #25 (PKCS#8 encrypted SHA1-3DES DER, 4096-bit)', - 'Parse RSA Key #26 (PKCS#8 encrypted SHA1-2DES)', - 'Parse RSA Key #26.1 (PKCS#8 encrypted SHA1-2DES, wrong PW)', - 'Parse RSA Key #26.2 (PKCS#8 encrypted SHA1-2DES, no PW)', - 'Parse RSA Key #27 (PKCS#8 encrypted SHA1-2DES, 2048-bit)', - 'Parse RSA Key #27.1 (PKCS#8 encrypted SHA1-2DES, 2048-bit, wrong PW)', - 'Parse RSA Key #27.2 (PKCS#8 encrypted SHA1-2DES, 2048-bit no PW)', - 'Parse RSA Key #28 (PKCS#8 encrypted SHA1-2DES, 4096-bit)', - 'Parse RSA Key #28.1 (PKCS#8 encrypted SHA1-2DES, 4096-bit, wrong PW)', - 'Parse RSA Key #28.2 (PKCS#8 encrypted SHA1-2DES, 4096-bit, no PW)', - 'Parse RSA Key #29 (PKCS#8 encrypted SHA1-2DES DER)', - 'Parse RSA Key #30 (PKCS#8 encrypted SHA1-2DES DER, 2048-bit)', - 'Parse RSA Key #31 (PKCS#8 encrypted SHA1-2DES DER, 4096-bit)', - 'Parse RSA Key #38 (PKCS#8 encrypted v2 PBKDF2 3DES)', - 'Parse RSA Key #38.1 (PKCS#8 encrypted v2 PBKDF2 3DES, wrong PW)', - 'Parse RSA Key #38.2 (PKCS#8 encrypted v2 PBKDF2 3DES, no PW)', - 'Parse RSA Key #39 (PKCS#8 encrypted v2 PBKDF2 3DES, 2048-bit)', - 'Parse RSA Key #39.1 (PKCS#8 encrypted v2 PBKDF2 3DES, 2048-bit, wrong PW)', - 'Parse RSA Key #39.2 (PKCS#8 encrypted v2 PBKDF2 3DES, 2048-bit, no PW)', - 'Parse RSA Key #40 (PKCS#8 encrypted v2 PBKDF2 3DES, 4096-bit)', - 'Parse RSA Key #40.1 (PKCS#8 encrypted v2 PBKDF2 3DES, 4096-bit, wrong PW)', - 'Parse RSA Key #40.2 (PKCS#8 encrypted v2 PBKDF2 3DES, 4096-bit, no PW)', - 'Parse RSA Key #41 (PKCS#8 encrypted v2 PBKDF2 3DES DER)', - 'Parse RSA Key #41.1 (PKCS#8 encrypted v2 PBKDF2 3DES DER, wrong PW)', - 'Parse RSA Key #41.2 (PKCS#8 encrypted v2 PBKDF2 3DES DER, no PW)', - 'Parse RSA Key #42 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 2048-bit)', - 'Parse RSA Key #42.1 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 2048-bit, wrong PW)', - 'Parse RSA Key #42.2 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 2048-bit, no PW)', - 'Parse RSA Key #43 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 4096-bit)', - 'Parse RSA Key #43.1 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 4096-bit, wrong PW)', - 'Parse RSA Key #43.2 (PKCS#8 encrypted v2 PBKDF2 3DES DER, 4096-bit, no PW)', - 'Parse RSA Key #44 (PKCS#8 encrypted v2 PBKDF2 DES)', - 'Parse RSA Key #44.1 (PKCS#8 encrypted v2 PBKDF2 DES, wrong PW)', - 'Parse RSA Key #44.2 (PKCS#8 encrypted v2 PBKDF2 DES, no PW)', - 'Parse RSA Key #45 (PKCS#8 encrypted v2 PBKDF2 DES, 2048-bit)', - 'Parse RSA Key #45.1 (PKCS#8 encrypted v2 PBKDF2 DES, 2048-bit, wrong PW)', - 'Parse RSA Key #45.2 (PKCS#8 encrypted v2 PBKDF2 DES, 2048-bit, no PW)', - 'Parse RSA Key #46 (PKCS#8 encrypted v2 PBKDF2 DES, 4096-bit)', - 'Parse RSA Key #46.1 (PKCS#8 encrypted v2 PBKDF2 DES, 4096-bit, wrong PW)', - 'Parse RSA Key #46.2 (PKCS#8 encrypted v2 PBKDF2 DES, 4096-bit, no PW)', - 'Parse RSA Key #47 (PKCS#8 encrypted v2 PBKDF2 DES DER)', - 'Parse RSA Key #47.1 (PKCS#8 encrypted v2 PBKDF2 DES DER, wrong PW)', - 'Parse RSA Key #47.2 (PKCS#8 encrypted v2 PBKDF2 DES DER, no PW)', - 'Parse RSA Key #48 (PKCS#8 encrypted v2 PBKDF2 DES DER, 2048-bit)', - 'Parse RSA Key #48.1 (PKCS#8 encrypted v2 PBKDF2 DES DER, 2048-bit, wrong PW)', - 'Parse RSA Key #48.2 (PKCS#8 encrypted v2 PBKDF2 DES DER, 2048-bit, no PW)', - 'Parse RSA Key #49 (PKCS#8 encrypted v2 PBKDF2 DES DER, 4096-bit)', - 'Parse RSA Key #49.1 (PKCS#8 encrypted v2 PBKDF2 DES DER, 4096-bit, wrong PW)', - 'Parse RSA Key #49.2 (PKCS#8 encrypted v2 PBKDF2 DES DER, 4096-bit, no PW)', - 'Parse RSA Key #50 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224)', - 'Parse RSA Key #50.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, wrong PW)', - 'Parse RSA Key #50.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, no PW)', - 'Parse RSA Key #51 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 2048-bit)', - 'Parse RSA Key #51.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 2048-bit, wrong PW)', - 'Parse RSA Key #51.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 2048-bit, no PW)', - 'Parse RSA Key #52 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 4096-bit)', - 'Parse RSA Key #52.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 4096-bit, wrong PW)', - 'Parse RSA Key #52.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224, 4096-bit, no PW)', - 'Parse RSA Key #53 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER)', - 'Parse RSA Key #53.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, wrong PW)', - 'Parse RSA Key #53.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, no PW)', - 'Parse RSA Key #54 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 2048-bit)', - 'Parse RSA Key #54.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #54.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 2048-bit, no PW)', - 'Parse RSA Key #55 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 4096-bit)', - 'Parse RSA Key #55.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #55.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA224 DER, 4096-bit, no PW)', - 'Parse RSA Key #56 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224)', - 'Parse RSA Key #56.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, wrong PW)', - 'Parse RSA Key #56.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, no PW)', - 'Parse RSA Key #57 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 2048-bit)', - 'Parse RSA Key #57.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 2048-bit, wrong PW)', - 'Parse RSA Key #57.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 2048-bit, no PW)', - 'Parse RSA Key #58 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 4096-bit)', - 'Parse RSA Key #58.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 4096-bit, wrong PW)', - 'Parse RSA Key #58.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224, 4096-bit, no PW)', - 'Parse RSA Key #59 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER)', - 'Parse RSA Key #59.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, wrong PW)', - 'Parse RSA Key #59.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, no PW)', - 'Parse RSA Key #60 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 2048-bit)', - 'Parse RSA Key #60.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #60.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 2048-bit, no PW)', - 'Parse RSA Key #61 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 4096-bit)', - 'Parse RSA Key #61.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #61.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA224 DER, 4096-bit, no PW)', - 'Parse RSA Key #62 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256)', - 'Parse RSA Key #62.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, wrong PW)', - 'Parse RSA Key #62.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, no PW)', - 'Parse RSA Key #63 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 2048-bit)', - 'Parse RSA Key #63.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 2048-bit, wrong PW)', - 'Parse RSA Key #63.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 2048-bit, no PW)', - 'Parse RSA Key #64 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 4096-bit)', - 'Parse RSA Key #64.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 4096-bit, wrong PW)', - 'Parse RSA Key #64.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256, 4096-bit, no PW)', - 'Parse RSA Key #65 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER)', - 'Parse RSA Key #65.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, wrong PW)', - 'Parse RSA Key #65.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, no PW)', - 'Parse RSA Key #66 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 2048-bit)', - 'Parse RSA Key #66.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #66.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 2048-bit, no PW)', - 'Parse RSA Key #67 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 4096-bit)', - 'Parse RSA Key #68.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #68.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA256 DER, 4096-bit, no PW)', - 'Parse RSA Key #69 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256)', - 'Parse RSA Key #69.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, wrong PW)', - 'Parse RSA Key #69.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, no PW)', - 'Parse RSA Key #70 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 2048-bit)', - 'Parse RSA Key #70.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 2048-bit, wrong PW)', - 'Parse RSA Key #70.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 2048-bit, no PW)', - 'Parse RSA Key #71 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 4096-bit)', - 'Parse RSA Key #71.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 4096-bit, wrong PW)', - 'Parse RSA Key #71.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256, 4096-bit, no PW)', - 'Parse RSA Key #72 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER)', - 'Parse RSA Key #72.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, wrong PW)', - 'Parse RSA Key #72.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, no PW)', - 'Parse RSA Key #73 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 2048-bit)', - 'Parse RSA Key #73.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #73.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 2048-bit, no PW)', - 'Parse RSA Key #74 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 4096-bit)', - 'Parse RSA Key #74.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #74.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA256 DER, 4096-bit, no PW)', - 'Parse RSA Key #75 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384)', - 'Parse RSA Key #75.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, wrong PW)', - 'Parse RSA Key #75.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, no PW)', - 'Parse RSA Key #76 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 2048-bit)', - 'Parse RSA Key #76.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 2048-bit, wrong PW)', - 'Parse RSA Key #76.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 2048-bit, no PW)', - 'Parse RSA Key #77 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 4096-bit)', - 'Parse RSA Key #77.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 4096-bit, wrong PW)', - 'Parse RSA Key #77.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384, 4096-bit, no PW)', - 'Parse RSA Key #78 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER)', - 'Parse RSA Key #78.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, wrong PW)', - 'Parse RSA Key #78.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, no PW)', - 'Parse RSA Key #79 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 2048-bit)', - 'Parse RSA Key #79.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #79.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 2048-bit, no PW)', - 'Parse RSA Key #80 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 4096-bit)', - 'Parse RSA Key #80.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #80.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA384 DER, 4096-bit, no PW)', - 'Parse RSA Key #81 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384)', - 'Parse RSA Key #81.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, wrong PW)', - 'Parse RSA Key #81.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, no PW)', - 'Parse RSA Key #82 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 2048-bit)', - 'Parse RSA Key #82.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 2048-bit, wrong PW)', - 'Parse RSA Key #82.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 2048-bit, no PW)', - 'Parse RSA Key #83 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 4096-bit)', - 'Parse RSA Key #83.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 4096-bit, wrong PW)', - 'Parse RSA Key #83.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384, 4096-bit, no PW)', - 'Parse RSA Key #84 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER)', - 'Parse RSA Key #84.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, wrong PW)', - 'Parse RSA Key #85.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, no PW)', - 'Parse RSA Key #86 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 2048-bit)', - 'Parse RSA Key #86.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #86.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 2048-bit, no PW)', - 'Parse RSA Key #87 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 4096-bit)', - 'Parse RSA Key #87.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #87.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA384 DER, 4096-bit, no PW)', - 'Parse RSA Key #88 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512)', - 'Parse RSA Key #88.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, wrong PW)', - 'Parse RSA Key #88.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, no PW)', - 'Parse RSA Key #89 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 2048-bit)', - 'Parse RSA Key #89.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 2048-bit, wrong PW)', - 'Parse RSA Key #89.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 2048-bit, no PW)', - 'Parse RSA Key #90 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 4096-bit)', - 'Parse RSA Key #90.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 4096-bit, wrong PW)', - 'Parse RSA Key #90.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512, 4096-bit, no PW)', - 'Parse RSA Key #91 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER)', - 'Parse RSA Key #91.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, wrong PW)', - 'Parse RSA Key #91.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, no PW)', - 'Parse RSA Key #92 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 2048-bit)', - 'Parse RSA Key #92.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #92.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 2048-bit, no PW)', - 'Parse RSA Key #93 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 4096-bit)', - 'Parse RSA Key #93.1 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #93.2 (PKCS#8 encrypted v2 PBKDF2 3DES hmacWithSHA512 DER, 4096-bit, no PW)', - 'Parse RSA Key #94 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512)', - 'Parse RSA Key #94.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, wrong PW)', - 'Parse RSA Key #94.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, no PW)', - 'Parse RSA Key #95 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 2048-bit)', - 'Parse RSA Key #95.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 2048-bit, wrong PW)', - 'Parse RSA Key #95.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 2048-bit, no PW)', - 'Parse RSA Key #96 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 4096-bit)', - 'Parse RSA Key #96.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 4096-bit, wrong PW)', - 'Parse RSA Key #96.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512, 4096-bit, no PW)', - 'Parse RSA Key #97 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER)', - 'Parse RSA Key #97.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, wrong PW)', - 'Parse RSA Key #97.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, no PW)', - 'Parse RSA Key #98 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 2048-bit)', - 'Parse RSA Key #98.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 2048-bit, wrong PW)', - 'Parse RSA Key #98.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 2048-bit, no PW)', - 'Parse RSA Key #99 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 4096-bit)', - 'Parse RSA Key #99.1 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 4096-bit, wrong PW)', - 'Parse RSA Key #99.2 (PKCS#8 encrypted v2 PBKDF2 DES hmacWithSHA512 DER, 4096-bit, no PW)', + re.compile(r'Parse RSA Key .*\(PKCS#8 encrypted .*\)'), ], } } @@ -517,11 +293,13 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', 'ignored_suites': [ - 'ecdsa', - 'ecdh', - 'ecjpake', + # Modules replaced by drivers + 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { + # This test wants a legacy function that takes f_rng, p_rng + # arguments, and uses legacy ECDSA for that. The test is + # really about the wrapper around the PSA RNG, not ECDSA. 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], @@ -529,49 +307,14 @@ KNOWN_TASKS = { # so we must ignore disparities in the tests for which ECP_C # is required. 'test_suite_ecp': [ - 'ECP check public-private #1 (OK)', - 'ECP check public-private #2 (group none)', - 'ECP check public-private #3 (group mismatch)', - 'ECP check public-private #4 (Qx mismatch)', - 'ECP check public-private #5 (Qy mismatch)', - 'ECP check public-private #6 (wrong Qx)', - 'ECP check public-private #7 (wrong Qy)', - 'ECP gen keypair [#1]', - 'ECP gen keypair [#2]', - 'ECP gen keypair [#3]', - 'ECP gen keypair wrapper', - 'ECP point muladd secp256r1 #1', - 'ECP point muladd secp256r1 #2', - 'ECP point multiplication Curve25519 (element of order 2: origin) #3', - 'ECP point multiplication Curve25519 (element of order 4: 1) #4', - 'ECP point multiplication Curve25519 (element of order 8) #5', - 'ECP point multiplication Curve25519 (normalized) #1', - 'ECP point multiplication Curve25519 (not normalized) #2', - 'ECP point multiplication rng fail Curve25519', - 'ECP point multiplication rng fail secp256r1', - 'ECP test vectors Curve25519', - 'ECP test vectors Curve448 (RFC 7748 6.2, after decodeUCoordinate)', - 'ECP test vectors brainpoolP256r1 rfc 7027', - 'ECP test vectors brainpoolP384r1 rfc 7027', - 'ECP test vectors brainpoolP512r1 rfc 7027', - 'ECP test vectors secp192k1', - 'ECP test vectors secp192r1 rfc 5114', - 'ECP test vectors secp224k1', - 'ECP test vectors secp224r1 rfc 5114', - 'ECP test vectors secp256k1', - 'ECP test vectors secp256r1 rfc 5114', - 'ECP test vectors secp384r1 rfc 5114', - 'ECP test vectors secp521r1 rfc 5114', - ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', + re.compile(r'ECP check public-private .*'), + re.compile(r'ECP gen keypair .*'), + re.compile(r'ECP point muladd .*'), + re.compile(r'ECP point multiplication .*'), + re.compile(r'ECP test vectors .*'), ], 'test_suite_ssl': [ + # This deprecated function is only present when ECP_C is On. 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], } @@ -583,32 +326,14 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - ], 'test_suite_pkparse': [ # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED # is automatically enabled in build_info.h (backward compatibility) @@ -616,23 +341,10 @@ KNOWN_TASKS = { # consequence compressed points are supported in the reference # component but not in the accelerated one, so they should be skipped # while checking driver's coverage. - 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', - 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', - 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', - 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', - 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', - 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', - 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', - 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', - 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', - 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', - 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', - 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', - 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', - 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', - 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', - 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), ], + # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], @@ -645,90 +357,31 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', - 'bignum_core', - 'bignum_random', - 'bignum_mod', - 'bignum_mod_raw', - 'bignum.generated', - 'bignum.misc', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - ], + # See no_ecp_at_all 'test_suite_pkparse': [ - # See the description provided above in the - # analyze_driver_vs_reference_no_ecp_at_all component. - 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', - 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', - 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', - 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', - 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', - 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', - 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', - 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', - 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', - 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', - 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', - 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', - 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', - 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', - 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', - 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), ], 'test_suite_asn1parse': [ - # This test depends on BIGNUM_C 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ - # Following tests depends on BIGNUM_C - 'ASN.1 Write mpi 0 (1 limb)', - 'ASN.1 Write mpi 0 (null)', - 'ASN.1 Write mpi 0x100', - 'ASN.1 Write mpi 0x7f', - 'ASN.1 Write mpi 0x7f with leading 0 limb', - 'ASN.1 Write mpi 0x80', - 'ASN.1 Write mpi 0x80 with leading 0 limb', - 'ASN.1 Write mpi 0xff', - 'ASN.1 Write mpi 1', - 'ASN.1 Write mpi, 127*8 bits', - 'ASN.1 Write mpi, 127*8+1 bits', - 'ASN.1 Write mpi, 127*8-1 bits', - 'ASN.1 Write mpi, 255*8 bits', - 'ASN.1 Write mpi, 255*8-1 bits', - 'ASN.1 Write mpi, 256*8-1 bits', + re.compile(r'ASN.1 Write mpi.*'), ], 'test_suite_debug': [ - # Following tests depends on BIGNUM_C - 'Debug print mbedtls_mpi #2: 3 bits', - 'Debug print mbedtls_mpi: 0 (empty representation)', - 'Debug print mbedtls_mpi: 0 (non-empty representation)', - 'Debug print mbedtls_mpi: 49 bits', - 'Debug print mbedtls_mpi: 759 bits', - 'Debug print mbedtls_mpi: 764 bits #1', - 'Debug print mbedtls_mpi: 764 bits #2', + re.compile(r'Debug print mbedtls_mpi.*'), ], + # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], @@ -741,91 +394,31 @@ KNOWN_TASKS = { 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', - 'bignum_core', - 'bignum_random', - 'bignum_mod', - 'bignum_mod_raw', - 'bignum.generated', - 'bignum.misc', - 'dhm', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto': [ - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - ], + # See no_ecp_at_all 'test_suite_pkparse': [ - # See the description provided above in the - # analyze_driver_vs_reference_no_ecp_at_all component. - 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', - 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', - 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', - 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', - 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', - 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', - 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', - 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', - 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', - 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', - 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', - 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', - 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', - 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', - 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', - 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), ], 'test_suite_asn1parse': [ - # This test depends on BIGNUM_C 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ - # Following tests depends on BIGNUM_C - 'ASN.1 Write mpi 0 (1 limb)', - 'ASN.1 Write mpi 0 (null)', - 'ASN.1 Write mpi 0x100', - 'ASN.1 Write mpi 0x7f', - 'ASN.1 Write mpi 0x7f with leading 0 limb', - 'ASN.1 Write mpi 0x80', - 'ASN.1 Write mpi 0x80 with leading 0 limb', - 'ASN.1 Write mpi 0xff', - 'ASN.1 Write mpi 1', - 'ASN.1 Write mpi, 127*8 bits', - 'ASN.1 Write mpi, 127*8+1 bits', - 'ASN.1 Write mpi, 127*8-1 bits', - 'ASN.1 Write mpi, 255*8 bits', - 'ASN.1 Write mpi, 255*8-1 bits', - 'ASN.1 Write mpi, 256*8-1 bits', + re.compile(r'ASN.1 Write mpi.*'), ], 'test_suite_debug': [ - # Following tests depends on BIGNUM_C - 'Debug print mbedtls_mpi #2: 3 bits', - 'Debug print mbedtls_mpi: 0 (empty representation)', - 'Debug print mbedtls_mpi: 0 (non-empty representation)', - 'Debug print mbedtls_mpi: 49 bits', - 'Debug print mbedtls_mpi: 759 bits', - 'Debug print mbedtls_mpi: 764 bits #1', - 'Debug print mbedtls_mpi: 764 bits #2', + re.compile(r'Debug print mbedtls_mpi.*'), ], + # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], @@ -847,92 +440,21 @@ KNOWN_TASKS = { 'component_ref': 'test_tfm_config', 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', 'ignored_suites': [ - # Ignore test suites for the modules that are disabled in the - # accelerated test case. - 'ecp', - 'ecdsa', - 'ecdh', - 'ecjpake', - 'bignum_core', - 'bignum_random', - 'bignum_mod', - 'bignum_mod_raw', - 'bignum.generated', - 'bignum.misc', + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { - # Ignore all tests that require DERIVE support which is disabled - # in the driver version - 'test_suite_psa_crypto': [ - 'PSA key agreement setup: ECDH + HKDF-SHA-256: good', - ('PSA key agreement setup: ECDH + HKDF-SHA-256: good, key algorithm broader ' - 'than required'), - 'PSA key agreement setup: ECDH + HKDF-SHA-256: public key not on curve', - 'PSA key agreement setup: KDF instead of a key agreement algorithm', - 'PSA key agreement setup: bad key agreement algorithm', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: capacity=8160', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 0+32', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 1+31', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 31+1', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 32+0', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 32+32', - 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 64+0', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, info first', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, key output', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, missing info', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, omitted salt', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, raw output', - 'PSA key derivation: ECDH on P256 with HKDF-SHA256, salt after secret', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, good case', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, missing label', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, missing label and secret', - 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, no inputs', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', - 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 0+48, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 24+24, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 48+0, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #1, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #3, ka', - 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #4, ka', - 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC MONTGOMERY (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', - 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', - 'PSA raw key agreement: ECDH SECP256R1 (RFC 5903)', - ], + # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], - 'test_suite_psa_crypto_pake': [ - 'PSA PAKE: ecjpake size macros', - ], 'test_suite_asn1parse': [ - # This test depends on BIGNUM_C 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ - # Following tests depends on BIGNUM_C - 'ASN.1 Write mpi 0 (1 limb)', - 'ASN.1 Write mpi 0 (null)', - 'ASN.1 Write mpi 0x100', - 'ASN.1 Write mpi 0x7f', - 'ASN.1 Write mpi 0x7f with leading 0 limb', - 'ASN.1 Write mpi 0x80', - 'ASN.1 Write mpi 0x80 with leading 0 limb', - 'ASN.1 Write mpi 0xff', - 'ASN.1 Write mpi 1', - 'ASN.1 Write mpi, 127*8 bits', - 'ASN.1 Write mpi, 127*8+1 bits', - 'ASN.1 Write mpi, 127*8-1 bits', - 'ASN.1 Write mpi, 255*8 bits', - 'ASN.1 Write mpi, 255*8-1 bits', - 'ASN.1 Write mpi, 256*8-1 bits', + re.compile(r'ASN.1 Write mpi.*'), ], } } @@ -971,7 +493,7 @@ def main(): tasks_list = re.split(r'[, ]+', options.specified_tasks) for task in tasks_list: if task not in KNOWN_TASKS: - sys.stderr.write('invalid task: {}'.format(task)) + sys.stderr.write('invalid task: {}\n'.format(task)) sys.exit(2) KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage