613 lines
26 KiB
Python
Executable file
613 lines
26 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
"""Analyze the test outcomes from a full CI run.
|
|
|
|
This script can also run on outcomes from a partial run, but the results are
|
|
less likely to be useful.
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
import traceback
|
|
import re
|
|
import subprocess
|
|
import os
|
|
import typing
|
|
|
|
import check_test_cases
|
|
|
|
|
|
# `ComponentOutcomes` is a named tuple which is defined as:
|
|
# ComponentOutcomes(
|
|
# successes = {
|
|
# "<suite_case>",
|
|
# ...
|
|
# },
|
|
# failures = {
|
|
# "<suite_case>",
|
|
# ...
|
|
# }
|
|
# )
|
|
# suite_case = "<suite>;<case>"
|
|
ComponentOutcomes = typing.NamedTuple('ComponentOutcomes',
|
|
[('successes', typing.Set[str]),
|
|
('failures', typing.Set[str])])
|
|
|
|
# `Outcomes` is a representation of the outcomes file,
|
|
# which defined as:
|
|
# Outcomes = {
|
|
# "<component>": ComponentOutcomes,
|
|
# ...
|
|
# }
|
|
Outcomes = typing.Dict[str, ComponentOutcomes]
|
|
|
|
|
|
class Results:
|
|
"""Process analysis results."""
|
|
|
|
def __init__(self):
|
|
self.error_count = 0
|
|
self.warning_count = 0
|
|
|
|
def new_section(self, fmt, *args, **kwargs):
|
|
self._print_line('\n*** ' + fmt + ' ***\n', *args, **kwargs)
|
|
|
|
def info(self, fmt, *args, **kwargs):
|
|
self._print_line('Info: ' + fmt, *args, **kwargs)
|
|
|
|
def error(self, fmt, *args, **kwargs):
|
|
self.error_count += 1
|
|
self._print_line('Error: ' + fmt, *args, **kwargs)
|
|
|
|
def warning(self, fmt, *args, **kwargs):
|
|
self.warning_count += 1
|
|
self._print_line('Warning: ' + fmt, *args, **kwargs)
|
|
|
|
@staticmethod
|
|
def _print_line(fmt, *args, **kwargs):
|
|
sys.stderr.write((fmt + '\n').format(*args, **kwargs))
|
|
|
|
def execute_reference_driver_tests(results: Results, ref_component: str, driver_component: str, \
|
|
outcome_file: str) -> None:
|
|
"""Run the tests specified in ref_component and driver_component. Results
|
|
are stored in the output_file and they will be used for the following
|
|
coverage analysis"""
|
|
results.new_section("Test {} and {}", ref_component, driver_component)
|
|
|
|
shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \
|
|
" " + ref_component + " " + driver_component
|
|
results.info("Running: {}", shell_command)
|
|
ret_val = subprocess.run(shell_command.split(), check=False).returncode
|
|
|
|
if ret_val != 0:
|
|
results.error("failed to run reference/driver components")
|
|
|
|
def analyze_coverage(results: Results, outcomes: Outcomes,
|
|
allow_list: typing.List[str], full_coverage: bool) -> None:
|
|
"""Check that all available test cases are executed at least once."""
|
|
available = check_test_cases.collect_available_test_cases()
|
|
for suite_case in available:
|
|
hit = any(suite_case in comp_outcomes.successes or
|
|
suite_case in comp_outcomes.failures
|
|
for comp_outcomes in outcomes.values())
|
|
|
|
if not hit and suite_case not in allow_list:
|
|
if full_coverage:
|
|
results.error('Test case not executed: {}', suite_case)
|
|
else:
|
|
results.warning('Test case not executed: {}', suite_case)
|
|
elif hit and suite_case in allow_list:
|
|
# Test Case should be removed from the allow list.
|
|
if full_coverage:
|
|
results.error('Allow listed test case was executed: {}', suite_case)
|
|
else:
|
|
results.warning('Allow listed test case was executed: {}', suite_case)
|
|
|
|
def name_matches_pattern(name: str, str_or_re) -> bool:
|
|
"""Check if name matches a pattern, that may be a string or regex.
|
|
- If the pattern is a string, name must be equal to match.
|
|
- If the pattern is a regex, name must fully match.
|
|
"""
|
|
# The CI's python is too old for re.Pattern
|
|
#if isinstance(str_or_re, re.Pattern):
|
|
if not isinstance(str_or_re, str):
|
|
return str_or_re.fullmatch(name) is not None
|
|
else:
|
|
return str_or_re == name
|
|
|
|
def analyze_driver_vs_reference(results: Results, outcomes: Outcomes,
|
|
component_ref: str, component_driver: str,
|
|
ignored_suites: typing.List[str], ignored_tests=None) -> None:
|
|
"""Check that all tests passing in the reference component are also
|
|
passing in the corresponding driver component.
|
|
Skip:
|
|
- full test suites provided in ignored_suites list
|
|
- only some specific test inside a test suite, for which the corresponding
|
|
output string is provided
|
|
"""
|
|
ref_outcomes = outcomes.get("component_" + component_ref)
|
|
driver_outcomes = outcomes.get("component_" + component_driver)
|
|
|
|
if ref_outcomes is None or driver_outcomes is None:
|
|
results.error("required components are missing: bad outcome file?")
|
|
return
|
|
|
|
if not ref_outcomes.successes:
|
|
results.error("no passing test in reference component: bad outcome file?")
|
|
return
|
|
|
|
for suite_case in ref_outcomes.successes:
|
|
# suite_case is like "test_suite_foo.bar;Description of test case"
|
|
(full_test_suite, test_string) = suite_case.split(';')
|
|
test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
|
|
|
|
# Immediately skip fully-ignored test suites
|
|
if test_suite in ignored_suites or full_test_suite in ignored_suites:
|
|
continue
|
|
|
|
# For ignored test cases inside test suites, just remember and:
|
|
# don't issue an error if they're skipped with drivers,
|
|
# but issue an error if they're not (means we have a bad entry).
|
|
ignored = False
|
|
if full_test_suite in ignored_tests:
|
|
for str_or_re in ignored_tests[test_suite]:
|
|
if name_matches_pattern(test_string, str_or_re):
|
|
ignored = True
|
|
|
|
if not ignored and not suite_case in driver_outcomes.successes:
|
|
results.error("PASS -> SKIP/FAIL: {}", suite_case)
|
|
if ignored and suite_case in driver_outcomes.successes:
|
|
results.error("uselessly ignored: {}", suite_case)
|
|
|
|
def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None:
|
|
"""Run all analyses on the given outcome collection."""
|
|
analyze_coverage(results, outcomes, args['allow_list'],
|
|
args['full_coverage'])
|
|
|
|
def read_outcome_file(outcome_file: str) -> Outcomes:
|
|
"""Parse an outcome file and return an outcome collection.
|
|
"""
|
|
outcomes = {}
|
|
with open(outcome_file, 'r', encoding='utf-8') as input_file:
|
|
for line in input_file:
|
|
(_platform, component, suite, case, result, _cause) = line.split(';')
|
|
# Note that `component` is not unique. If a test case passes on Linux
|
|
# and fails on FreeBSD, it'll end up in both the successes set and
|
|
# the failures set.
|
|
suite_case = ';'.join([suite, case])
|
|
if component not in outcomes:
|
|
outcomes[component] = ComponentOutcomes(set(), set())
|
|
if result == 'PASS':
|
|
outcomes[component].successes.add(suite_case)
|
|
elif result == 'FAIL':
|
|
outcomes[component].failures.add(suite_case)
|
|
|
|
return outcomes
|
|
|
|
def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None:
|
|
"""Perform coverage analysis."""
|
|
results.new_section("Analyze coverage")
|
|
analyze_outcomes(results, outcomes, args)
|
|
|
|
def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None:
|
|
"""Perform driver vs reference analyze."""
|
|
results.new_section("Analyze driver {} vs reference {}",
|
|
args['component_driver'], args['component_ref'])
|
|
|
|
ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
|
|
|
|
analyze_driver_vs_reference(results, outcomes,
|
|
args['component_ref'], args['component_driver'],
|
|
ignored_suites, args['ignored_tests'])
|
|
|
|
# List of tasks with a function that can handle this task and additional arguments if required
|
|
KNOWN_TASKS = {
|
|
'analyze_coverage': {
|
|
'test_function': do_analyze_coverage,
|
|
'args': {
|
|
'allow_list': [
|
|
# Algorithm not supported yet
|
|
'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
|
|
# Algorithm not supported yet
|
|
'test_suite_psa_crypto_metadata;Cipher: XTS',
|
|
],
|
|
'full_coverage': False,
|
|
}
|
|
},
|
|
# There are 2 options to use analyze_driver_vs_reference_xxx locally:
|
|
# 1. Run tests and then analysis:
|
|
# - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
|
|
# - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
|
|
# 2. Let this script run both automatically:
|
|
# - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
|
|
'analyze_driver_vs_reference_hash': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_hash_use_psa',
|
|
'component_driver': 'test_psa_crypto_config_accel_hash_use_psa',
|
|
'ignored_suites': [
|
|
'shax', 'mdx', # the software implementations that are being excluded
|
|
'md.psa', # purposefully depends on whether drivers are present
|
|
'psa_crypto_low_hash.generated', # testing the builtins
|
|
],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_cipher_aead': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_cipher_aead',
|
|
'component_driver': 'test_psa_crypto_config_accel_cipher_aead',
|
|
# Modules replaced by drivers.
|
|
'ignored_suites': [
|
|
# low-level (block/stream) cipher modules
|
|
'aes', 'aria', 'camellia', 'des', 'chacha20',
|
|
# AEAD modes
|
|
'ccm', 'chachapoly', 'cmac', 'gcm',
|
|
# The Cipher abstraction layer
|
|
'cipher',
|
|
],
|
|
'ignored_tests': {
|
|
# PEM decryption is not supported so far.
|
|
# The rest of PEM (write, unencrypted read) works though.
|
|
'test_suite_pem': [
|
|
re.compile(r'PEM read .*(AES|DES|\bencrypt).*'),
|
|
],
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
# Following tests depend on AES_C/DES_C but are not about
|
|
# them really, just need to know some error code is there.
|
|
'test_suite_error': [
|
|
'Low and high error',
|
|
'Single low error'
|
|
],
|
|
# Similar to test_suite_error above.
|
|
'test_suite_version': [
|
|
'Check for MBEDTLS_AES_C when already present',
|
|
],
|
|
# The en/decryption part of PKCS#12 is not supported so far.
|
|
# The rest of PKCS#12 (key derivation) works though.
|
|
'test_suite_pkcs12': [
|
|
re.compile(r'PBE Encrypt, .*'),
|
|
re.compile(r'PBE Decrypt, .*'),
|
|
],
|
|
# The en/decryption part of PKCS#5 is not supported so far.
|
|
# The rest of PKCS#5 (PBKDF2) works though.
|
|
'test_suite_pkcs5': [
|
|
re.compile(r'PBES2 Encrypt, .*'),
|
|
re.compile(r'PBES2 Decrypt .*'),
|
|
],
|
|
# Encrypted keys are not supported so far.
|
|
# pylint: disable=line-too-long
|
|
'test_suite_pkparse': [
|
|
'Key ASN1 (Encrypted key PKCS12, trailing garbage data)',
|
|
'Key ASN1 (Encrypted key PKCS5, trailing garbage data)',
|
|
re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'),
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_ecp_light_only': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only',
|
|
'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only',
|
|
'ignored_suites': [
|
|
# Modules replaced by drivers
|
|
'ecdsa', 'ecdh', 'ecjpake',
|
|
],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
# This test wants a legacy function that takes f_rng, p_rng
|
|
# arguments, and uses legacy ECDSA for that. The test is
|
|
# really about the wrapper around the PSA RNG, not ECDSA.
|
|
'test_suite_random': [
|
|
'PSA classic wrapper: ECDSA signature (SECP256R1)',
|
|
],
|
|
# In the accelerated test ECP_C is not set (only ECP_LIGHT is)
|
|
# so we must ignore disparities in the tests for which ECP_C
|
|
# is required.
|
|
'test_suite_ecp': [
|
|
re.compile(r'ECP check public-private .*'),
|
|
re.compile(r'ECP gen keypair .*'),
|
|
re.compile(r'ECP point muladd .*'),
|
|
re.compile(r'ECP point multiplication .*'),
|
|
re.compile(r'ECP test vectors .*'),
|
|
],
|
|
'test_suite_ssl': [
|
|
# This deprecated function is only present when ECP_C is On.
|
|
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_no_ecp_at_all': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all',
|
|
'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all',
|
|
'ignored_suites': [
|
|
# Modules replaced by drivers
|
|
'ecp', 'ecdsa', 'ecdh', 'ecjpake',
|
|
],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
# See ecp_light_only
|
|
'test_suite_random': [
|
|
'PSA classic wrapper: ECDSA signature (SECP256R1)',
|
|
],
|
|
'test_suite_pkparse': [
|
|
# When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
|
|
# is automatically enabled in build_info.h (backward compatibility)
|
|
# even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
|
|
# consequence compressed points are supported in the reference
|
|
# component but not in the accelerated one, so they should be skipped
|
|
# while checking driver's coverage.
|
|
re.compile(r'Parse EC Key .*compressed\)'),
|
|
re.compile(r'Parse Public EC Key .*compressed\)'),
|
|
],
|
|
# See ecp_light_only
|
|
'test_suite_ssl': [
|
|
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_ecc_no_bignum': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum',
|
|
'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum',
|
|
'ignored_suites': [
|
|
# Modules replaced by drivers
|
|
'ecp', 'ecdsa', 'ecdh', 'ecjpake',
|
|
'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
|
|
'bignum.generated', 'bignum.misc',
|
|
],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
# See ecp_light_only
|
|
'test_suite_random': [
|
|
'PSA classic wrapper: ECDSA signature (SECP256R1)',
|
|
],
|
|
# See no_ecp_at_all
|
|
'test_suite_pkparse': [
|
|
re.compile(r'Parse EC Key .*compressed\)'),
|
|
re.compile(r'Parse Public EC Key .*compressed\)'),
|
|
],
|
|
'test_suite_asn1parse': [
|
|
'INTEGER too large for mpi',
|
|
],
|
|
'test_suite_asn1write': [
|
|
re.compile(r'ASN.1 Write mpi.*'),
|
|
],
|
|
'test_suite_debug': [
|
|
re.compile(r'Debug print mbedtls_mpi.*'),
|
|
],
|
|
# See ecp_light_only
|
|
'test_suite_ssl': [
|
|
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_ecc_ffdh_no_bignum': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum',
|
|
'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum',
|
|
'ignored_suites': [
|
|
# Modules replaced by drivers
|
|
'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
|
|
'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
|
|
'bignum.generated', 'bignum.misc',
|
|
],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
# See ecp_light_only
|
|
'test_suite_random': [
|
|
'PSA classic wrapper: ECDSA signature (SECP256R1)',
|
|
],
|
|
# See no_ecp_at_all
|
|
'test_suite_pkparse': [
|
|
re.compile(r'Parse EC Key .*compressed\)'),
|
|
re.compile(r'Parse Public EC Key .*compressed\)'),
|
|
],
|
|
'test_suite_asn1parse': [
|
|
'INTEGER too large for mpi',
|
|
],
|
|
'test_suite_asn1write': [
|
|
re.compile(r'ASN.1 Write mpi.*'),
|
|
],
|
|
'test_suite_debug': [
|
|
re.compile(r'Debug print mbedtls_mpi.*'),
|
|
],
|
|
# See ecp_light_only
|
|
'test_suite_ssl': [
|
|
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_ffdh_alg': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_ffdh',
|
|
'component_driver': 'test_psa_crypto_config_accel_ffdh',
|
|
'ignored_suites': ['dhm'],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_tfm_config': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_tfm_config',
|
|
'component_driver': 'test_tfm_config_p256m_driver_accel_ec',
|
|
'ignored_suites': [
|
|
# Modules replaced by drivers
|
|
'asn1parse', 'asn1write',
|
|
'ecp', 'ecdsa', 'ecdh', 'ecjpake',
|
|
'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
|
|
'bignum.generated', 'bignum.misc',
|
|
],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
# See ecp_light_only
|
|
'test_suite_random': [
|
|
'PSA classic wrapper: ECDSA signature (SECP256R1)',
|
|
],
|
|
}
|
|
}
|
|
},
|
|
'analyze_driver_vs_reference_rsa': {
|
|
'test_function': do_analyze_driver_vs_reference,
|
|
'args': {
|
|
'component_ref': 'test_psa_crypto_config_reference_rsa_crypto',
|
|
'component_driver': 'test_psa_crypto_config_accel_rsa_crypto',
|
|
'ignored_suites': [
|
|
# Modules replaced by drivers.
|
|
'rsa', 'pkcs1_v15', 'pkcs1_v21',
|
|
# We temporarily don't care about PK stuff.
|
|
'pk', 'pkwrite', 'pkparse'
|
|
],
|
|
'ignored_tests': {
|
|
'test_suite_platform': [
|
|
# Incompatible with sanitizers (e.g. ASan). If the driver
|
|
# component uses a sanitizer but the reference component
|
|
# doesn't, we have a PASS vs SKIP mismatch.
|
|
'Check mbedtls_calloc overallocation',
|
|
],
|
|
# Following tests depend on RSA_C but are not about
|
|
# them really, just need to know some error code is there.
|
|
'test_suite_error': [
|
|
'Low and high error',
|
|
'Single high error'
|
|
],
|
|
# Constant time operations only used for PKCS1_V15
|
|
'test_suite_constant_time': [
|
|
re.compile(r'mbedtls_ct_zeroize_if .*'),
|
|
re.compile(r'mbedtls_ct_memmove_left .*')
|
|
],
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
def main():
|
|
main_results = Results()
|
|
|
|
try:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('outcomes', metavar='OUTCOMES.CSV',
|
|
help='Outcome file to analyze')
|
|
parser.add_argument('specified_tasks', default='all', nargs='?',
|
|
help='Analysis to be done. By default, run all tasks. '
|
|
'With one or more TASK, run only those. '
|
|
'TASK can be the name of a single task or '
|
|
'comma/space-separated list of tasks. ')
|
|
parser.add_argument('--list', action='store_true',
|
|
help='List all available tasks and exit.')
|
|
parser.add_argument('--require-full-coverage', action='store_true',
|
|
dest='full_coverage', help="Require all available "
|
|
"test cases to be executed and issue an error "
|
|
"otherwise. This flag is ignored if 'task' is "
|
|
"neither 'all' nor 'analyze_coverage'")
|
|
options = parser.parse_args()
|
|
|
|
if options.list:
|
|
for task in KNOWN_TASKS:
|
|
print(task)
|
|
sys.exit(0)
|
|
|
|
if options.specified_tasks == 'all':
|
|
tasks_list = KNOWN_TASKS.keys()
|
|
else:
|
|
tasks_list = re.split(r'[, ]+', options.specified_tasks)
|
|
for task in tasks_list:
|
|
if task not in KNOWN_TASKS:
|
|
sys.stderr.write('invalid task: {}\n'.format(task))
|
|
sys.exit(2)
|
|
|
|
KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage
|
|
|
|
# If the outcome file exists, parse it once and share the result
|
|
# among tasks to improve performance.
|
|
# Otherwise, it will be generated by execute_reference_driver_tests.
|
|
if not os.path.exists(options.outcomes):
|
|
if len(tasks_list) > 1:
|
|
sys.stderr.write("mutiple tasks found, please provide a valid outcomes file.\n")
|
|
sys.exit(2)
|
|
|
|
task_name = tasks_list[0]
|
|
task = KNOWN_TASKS[task_name]
|
|
if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable
|
|
sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name))
|
|
sys.exit(2)
|
|
|
|
execute_reference_driver_tests(main_results,
|
|
task['args']['component_ref'],
|
|
task['args']['component_driver'],
|
|
options.outcomes)
|
|
|
|
outcomes = read_outcome_file(options.outcomes)
|
|
|
|
for task in tasks_list:
|
|
test_function = KNOWN_TASKS[task]['test_function']
|
|
test_args = KNOWN_TASKS[task]['args']
|
|
test_function(main_results, outcomes, test_args)
|
|
|
|
main_results.info("Overall results: {} warnings and {} errors",
|
|
main_results.warning_count, main_results.error_count)
|
|
|
|
sys.exit(0 if (main_results.error_count == 0) else 1)
|
|
|
|
except Exception: # pylint: disable=broad-except
|
|
# Print the backtrace and exit explicitly with our chosen status.
|
|
traceback.print_exc()
|
|
sys.exit(120)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|