test: extend analyze_outcomes.py in order to skip only some test in a suite

Signed-off-by: Valerio Setti <valerio.setti@nordicsemi.no>

tests/scripts/analyze_outcomes.py

@@ -61,24 +61,32 @@ def analyze_coverage(results, outcomes):
             # fixed this branch to have full coverage of test cases.
             results.warning('Test case not executed: {}', key)
 
-def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests):
+def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
+                                ignored_suites, ignored_test=None):
     """Check that all tests executed in the reference component are also
     executed in the corresponding driver component.
-    Skip test suites provided in ignored_tests list.
+    Skip:
+    - full test suites provided in ignored_suites list
+    - only some specific test inside a test suite, for which the corresponding
+      output string is provided
     """
     available = check_test_cases.collect_available_test_cases()
     result = True
 
     for key in available:
         # Skip ignored test suites
-        test_suite = key.split(';')[0] # retrieve test suit name
+        full_test_suite = key.split(';')[0] # retrieve full test suit name
-        test_suite = test_suite.split('.')[0] # retrieve main part of test suit name
+        test_string = key.split(';')[1] # retrieve the text string of this test
-        if test_suite in ignored_tests:
+        test_suite = full_test_suite.split('.')[0] # retrieve main part of test suit name
+        if test_suite in ignored_suites:
             continue
         # Continue if test was not executed by any component
         hits = outcomes[key].hits() if key in outcomes else 0
         if hits == 0:
             continue
+        if ((full_test_suite in ignored_test) and
+                (test_string in ignored_test[full_test_suite])):
+            continue
         # Search for tests that run in reference component and not in driver component
         driver_test_passed = False
         reference_test_passed = False
@@ -129,13 +137,14 @@ def do_analyze_coverage(outcome_file, args):
 
 def do_analyze_driver_vs_reference(outcome_file, args):
     """Perform driver vs reference analyze."""
-    ignored_tests = ['test_suite_' + x for x in args['ignored_suites']]
+    ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
 
     outcomes = read_outcome_file(outcome_file)
     print("\n*** Analyze driver {} vs reference {} ***\n".format(
         args['component_driver'], args['component_ref']))
     return analyze_driver_vs_reference(outcomes, args['component_ref'],
-                                       args['component_driver'], ignored_tests)
+                                       args['component_driver'], ignored_suites,
+                                       args['ignored_tests'])
 
 # List of tasks with a function that can handle this task and additional arguments if required
 TASKS = {
@@ -154,7 +163,11 @@ TASKS = {
             'ignored_suites': [
                 'shax', 'mdx', # the software implementations that are being excluded
                 'md',  # the legacy abstraction layer that's being excluded
-            ]}},
+            ],
+            'ignored_tests': {
+            }
+        }
+    },
     'analyze_driver_vs_reference_ecdsa': {
         'test_function': do_analyze_driver_vs_reference,
         'args': {
@@ -170,7 +183,11 @@ TASKS = {
                 'x509write', # #6858
                 'debug', # #6860
                 'ssl', # #6860
-            ]}},
+            ],
+            'ignored_tests': {
+            }
+        }
+    },
 }
 
 def main():