yuzu-emu/mbedtls
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

test: extend analyze_outcomes.py in order to skip only some test in a suite

Browse source 
Signed-off-by: Valerio Setti <valerio.setti@nordicsemi.no>
This commit is contained in:
Valerio Setti 2023-01-18 17:28:36 +01:00
parent 7e57920511
commit 3002c99247
1 changed files with 26 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
tests/scripts/analyze_outcomes.py
View file

@ -61,24 +61,32 @@ def analyze_coverage(results, outcomes):
# fixed this branch to have full coverage of test cases.
results.warning('Test case not executed: {}', key)
def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests):
def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
ignored_suites, ignored_test=None):
"""Check that all tests executed in the reference component are also
executed in the corresponding driver component.
Skip test suites provided in ignored_tests list.
Skip:
- full test suites provided in ignored_suites list
- only some specific test inside a test suite, for which the corresponding
output string is provided
"""
available = check_test_cases.collect_available_test_cases()
result = True
for key in available:
# Skip ignored test suites
test_suite = key.split(';')[0] # retrieve test suit name
test_suite = test_suite.split('.')[0] # retrieve main part of test suit name
if test_suite in ignored_tests:
full_test_suite = key.split(';')[0] # retrieve full test suit name
test_string = key.split(';')[1] # retrieve the text string of this test
test_suite = full_test_suite.split('.')[0] # retrieve main part of test suit name
if test_suite in ignored_suites:
continue
# Continue if test was not executed by any component
hits = outcomes[key].hits() if key in outcomes else 0
if hits == 0:
continue
if ((full_test_suite in ignored_test) and
(test_string in ignored_test[full_test_suite])):
continue
# Search for tests that run in reference component and not in driver component
driver_test_passed = False
reference_test_passed = False
@ -129,13 +137,14 @@ def do_analyze_coverage(outcome_file, args):
def do_analyze_driver_vs_reference(outcome_file, args):
"""Perform driver vs reference analyze."""
ignored_tests = ['test_suite_' + x for x in args['ignored_suites']]
ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
outcomes = read_outcome_file(outcome_file)
print("\n*** Analyze driver {} vs reference {} ***\n".format(
args['component_driver'], args['component_ref']))
return analyze_driver_vs_reference(outcomes, args['component_ref'],
args['component_driver'], ignored_tests)
args['component_driver'], ignored_suites,
args['ignored_tests'])
# List of tasks with a function that can handle this task and additional arguments if required
TASKS = {
@ -154,7 +163,11 @@ TASKS = {
'ignored_suites': [
'shax', 'mdx', # the software implementations that are being excluded
'md', # the legacy abstraction layer that's being excluded
]}},
],
'ignored_tests': {
}
}
},
'analyze_driver_vs_reference_ecdsa': {
'test_function': do_analyze_driver_vs_reference,
'args': {
@ -170,7 +183,11 @@ TASKS = {
'x509write', # #6858
'debug', # #6860
'ssl', # #6860
]}},
],
'ignored_tests': {
}
}
},
}
def main():