yuzu-emu/mbedtls
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #8074 from tgonzalezorlandoarm/tg/allowlist

Browse source 
Implement allowlist of test cases that are legitimately not executed
This commit is contained in:
Gilles Peskine 2023-08-24 18:03:20 +00:00 committed by GitHub
commit 8ca2041145
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 34 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
tests/scripts/analyze_outcomes.py
View file

@ -73,15 +73,22 @@ def execute_reference_driver_tests(ref_component, driver_component, outcome_file
Results.log("Error: failed to run reference/driver components") Results.log("Error: failed to run reference/driver components")
sys.exit(ret_val) sys.exit(ret_val)
def analyze_coverage(results, outcomes): def analyze_coverage(results, outcomes, allow_list, full_coverage):
"""Check that all available test cases are executed at least once.""" """Check that all available test cases are executed at least once."""
available = check_test_cases.collect_available_test_cases() available = check_test_cases.collect_available_test_cases()
for key in available: for key in available:
hits = outcomes[key].hits() if key in outcomes else 0 hits = outcomes[key].hits() if key in outcomes else 0
if hits == 0: if hits == 0 and key not in allow_list:
# Make this a warning, not an error, as long as we haven't if full_coverage:
# fixed this branch to have full coverage of test cases. results.error('Test case not executed: {}', key)
results.warning('Test case not executed: {}', key) else:
results.warning('Test case not executed: {}', key)
elif hits != 0 and key in allow_list:
# Test Case should be removed from the allow list.
if full_coverage:
results.error('Allow listed test case was executed: {}', key)
else:
results.warning('Allow listed test case was executed: {}', key)
def analyze_driver_vs_reference(outcomes, component_ref, component_driver, def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
ignored_suites, ignored_test=None): ignored_suites, ignored_test=None):
@ -122,10 +129,11 @@ def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
result = False result = False
return result return result
def analyze_outcomes(outcomes): def analyze_outcomes(outcomes, args):
"""Run all analyses on the given outcome collection.""" """Run all analyses on the given outcome collection."""
results = Results() results = Results()
analyze_coverage(results, outcomes) analyze_coverage(results, outcomes, args['allow_list'],
args['full_coverage'])
return results return results
def read_outcome_file(outcome_file): def read_outcome_file(outcome_file):
@ -151,10 +159,9 @@ by a semicolon.
def do_analyze_coverage(outcome_file, args): def do_analyze_coverage(outcome_file, args):
"""Perform coverage analysis.""" """Perform coverage analysis."""
del args # unused
outcomes = read_outcome_file(outcome_file) outcomes = read_outcome_file(outcome_file)
Results.log("\n*** Analyze coverage ***\n") Results.log("\n*** Analyze coverage ***\n")
results = analyze_outcomes(outcomes) results = analyze_outcomes(outcomes, args)
return results.error_count == 0 return results.error_count == 0
def do_analyze_driver_vs_reference(outcome_file, args): def do_analyze_driver_vs_reference(outcome_file, args):
@ -175,8 +182,16 @@ def do_analyze_driver_vs_reference(outcome_file, args):
TASKS = { TASKS = {
'analyze_coverage': { 'analyze_coverage': {
'test_function': do_analyze_coverage, 'test_function': do_analyze_coverage,
'args': {} 'args': {
}, 'allow_list': [
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Cipher: XTS',
],
'full_coverage': False,
}
},
# There are 2 options to use analyze_driver_vs_reference_xxx locally: # There are 2 options to use analyze_driver_vs_reference_xxx locally:
# 1. Run tests and then analysis: # 1. Run tests and then analysis:
# - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver> # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
@ -426,6 +441,11 @@ def main():
'comma/space-separated list of tasks. ') 'comma/space-separated list of tasks. ')
parser.add_argument('--list', action='store_true', parser.add_argument('--list', action='store_true',
help='List all available tasks and exit.') help='List all available tasks and exit.')
parser.add_argument('--require-full-coverage', action='store_true',
dest='full_coverage', help="Require all available "
"test cases to be executed and issue an error "
"otherwise. This flag is ignored if 'task' is "
"neither 'all' nor 'analyze_coverage'")
options = parser.parse_args() options = parser.parse_args()
if options.list: if options.list:
@ -445,6 +465,9 @@ def main():
Results.log('Error: invalid task: {}'.format(task)) Results.log('Error: invalid task: {}'.format(task))
sys.exit(1) sys.exit(1)
TASKS['analyze_coverage']['args']['full_coverage'] = \
options.full_coverage
for task in TASKS: for task in TASKS:
if task in tasks: if task in tasks:
if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']): if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']):