diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index fde07159e..24f4da773 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -73,15 +73,16 @@ def execute_reference_driver_tests(ref_component, driver_component, outcome_file Results.log("Error: failed to run reference/driver components") sys.exit(ret_val) -def analyze_coverage(results, outcomes, allow_list): +def analyze_coverage(results, outcomes, allow_list, full_coverage): """Check that all available test cases are executed at least once.""" available = check_test_cases.collect_available_test_cases() for key in available: hits = outcomes[key].hits() if key in outcomes else 0 if hits == 0 and key not in allow_list: - # Make this a warning, not an error, as long as we haven't - # fixed this branch to have full coverage of test cases. - results.warning('Test case not executed: {}', key) + if full_coverage: + results.error('Test case not executed: {}', key) + else: + results.warning('Test case not executed: {}', key) elif hits != 0 and key in allow_list: # Test Case should be removed from the allow list. results.warning('Allow listed test case was executed: {}', key) @@ -125,10 +126,11 @@ def analyze_driver_vs_reference(outcomes, component_ref, component_driver, result = False return result -def analyze_outcomes(outcomes, allow_list): +def analyze_outcomes(outcomes, args): """Run all analyses on the given outcome collection.""" results = Results() - analyze_coverage(results, outcomes, allow_list) + analyze_coverage(results, outcomes, args['allow_list'], + args['full_coverage']) return results def read_outcome_file(outcome_file): @@ -156,7 +158,7 @@ def do_analyze_coverage(outcome_file, args): """Perform coverage analysis.""" outcomes = read_outcome_file(outcome_file) Results.log("\n*** Analyze coverage ***\n") - results = analyze_outcomes(outcomes, args['allow_list']) + results = analyze_outcomes(outcomes, args) return results.error_count == 0 def do_analyze_driver_vs_reference(outcome_file, args): @@ -179,6 +181,7 @@ TASKS = { 'test_function': do_analyze_coverage, 'args': { 'allow_list': [], + 'full_coverage': False, } }, # There are 2 options to use analyze_driver_vs_reference_xxx locally: @@ -430,6 +433,11 @@ def main(): 'comma/space-separated list of tasks. ') parser.add_argument('--list', action='store_true', help='List all available tasks and exit.') + parser.add_argument('--require-full-coverage', action='store_true', + dest='full_coverage', help="Require all available " + "test cases to be executed and issue an error " + "otherwise. This flag is ignored if 'task' is " + "neither 'all' nor 'analyze_coverage'") options = parser.parse_args() if options.list: @@ -449,6 +457,9 @@ def main(): Results.log('Error: invalid task: {}'.format(task)) sys.exit(1) + TASKS['analyze_coverage']['args']['full_coverage'] = \ + options.full_coverage + for task in TASKS: if task in tasks: if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']):