yuzu-emu/mbedtls
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add a flag for requiring full coverage in coverage tests

Browse source 
Introduce the --require-full-coverage in analyze_outcomes.py so that
when analyze_outcomes.py --require-full-coverage is called, those
tests that are not executed and are not in the allowed list issue an
error instead of a warning.

Note that it is useful to run analyze_outcomes.py on incomplete test
results, so this error mode needs to remain optional in the long
term.

Signed-off-by: Tomás González <tomasagustin.gonzalezorlando@arm.com>
This commit is contained in:
Tomás González 2023-08-11 15:22:04 +01:00
parent 07bdcc2b0d
commit b401e113ff
1 changed files with 18 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
tests/scripts/analyze_outcomes.py
View file

@ -73,15 +73,16 @@ def execute_reference_driver_tests(ref_component, driver_component, outcome_file
Results.log("Error: failed to run reference/driver components") Results.log("Error: failed to run reference/driver components")
sys.exit(ret_val) sys.exit(ret_val)
def analyze_coverage(results, outcomes, allow_list): def analyze_coverage(results, outcomes, allow_list, full_coverage):
"""Check that all available test cases are executed at least once.""" """Check that all available test cases are executed at least once."""
available = check_test_cases.collect_available_test_cases() available = check_test_cases.collect_available_test_cases()
for key in available: for key in available:
hits = outcomes[key].hits() if key in outcomes else 0 hits = outcomes[key].hits() if key in outcomes else 0
if hits == 0 and key not in allow_list: if hits == 0 and key not in allow_list:
# Make this a warning, not an error, as long as we haven't if full_coverage:
# fixed this branch to have full coverage of test cases. results.error('Test case not executed: {}', key)
results.warning('Test case not executed: {}', key) else:
results.warning('Test case not executed: {}', key)
elif hits != 0 and key in allow_list: elif hits != 0 and key in allow_list:
# Test Case should be removed from the allow list. # Test Case should be removed from the allow list.
results.warning('Allow listed test case was executed: {}', key) results.warning('Allow listed test case was executed: {}', key)
@ -125,10 +126,11 @@ def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
result = False result = False
return result return result
def analyze_outcomes(outcomes, allow_list): def analyze_outcomes(outcomes, args):
"""Run all analyses on the given outcome collection.""" """Run all analyses on the given outcome collection."""
results = Results() results = Results()
analyze_coverage(results, outcomes, allow_list) analyze_coverage(results, outcomes, args['allow_list'],
args['full_coverage'])
return results return results
def read_outcome_file(outcome_file): def read_outcome_file(outcome_file):
@ -156,7 +158,7 @@ def do_analyze_coverage(outcome_file, args):
"""Perform coverage analysis.""" """Perform coverage analysis."""
outcomes = read_outcome_file(outcome_file) outcomes = read_outcome_file(outcome_file)
Results.log("\n*** Analyze coverage ***\n") Results.log("\n*** Analyze coverage ***\n")
results = analyze_outcomes(outcomes, args['allow_list']) results = analyze_outcomes(outcomes, args)
return results.error_count == 0 return results.error_count == 0
def do_analyze_driver_vs_reference(outcome_file, args): def do_analyze_driver_vs_reference(outcome_file, args):
@ -179,6 +181,7 @@ TASKS = {
'test_function': do_analyze_coverage, 'test_function': do_analyze_coverage,
'args': { 'args': {
'allow_list': [], 'allow_list': [],
'full_coverage': False,
} }
}, },
# There are 2 options to use analyze_driver_vs_reference_xxx locally: # There are 2 options to use analyze_driver_vs_reference_xxx locally:
@ -430,6 +433,11 @@ def main():
'comma/space-separated list of tasks. ') 'comma/space-separated list of tasks. ')
parser.add_argument('--list', action='store_true', parser.add_argument('--list', action='store_true',
help='List all available tasks and exit.') help='List all available tasks and exit.')
parser.add_argument('--require-full-coverage', action='store_true',
dest='full_coverage', help="Require all available "
"test cases to be executed and issue an error "
"otherwise. This flag is ignored if 'task' is "
"neither 'all' nor 'analyze_coverage'")
options = parser.parse_args() options = parser.parse_args()
if options.list: if options.list:
@ -449,6 +457,9 @@ def main():
Results.log('Error: invalid task: {}'.format(task)) Results.log('Error: invalid task: {}'.format(task))
sys.exit(1) sys.exit(1)
TASKS['analyze_coverage']['args']['full_coverage'] = \
options.full_coverage
for task in TASKS: for task in TASKS:
if task in tasks: if task in tasks:
if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']): if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']):