From c51c411cc1228a6916c2ba44ac1135cfdca37725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Mon, 30 Oct 2023 10:21:22 +0100 Subject: [PATCH] analyze_outcome: only warn on ignored tests that pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous check also warned when on tests that were already skipped in the reference config, which are not really a problem. The purpose of this "uselessly ignored" check is to make sure that the ignore list (together with the config common to driver and reference in all.sh) always correct reflects what works or doesn't in driver-only builds. For this it's enough to warn when a test is ignored but passing. The previous, stricter check, was causing issues like: Error: uselessly ignored: test_suite_pkcs12;PBE Encrypt, pad = 8 (PKCS7 padding disabled) Error: uselessly ignored: test_suite_pkcs12;PBE Decrypt, (Invalid padding & PKCS7 padding disabled) Error: uselessly ignored: test_suite_pkcs5;PBES2 Decrypt (Invalid padding & PKCS7 padding disabled) These are skipped in the reference config because is has PKCS7 padding enabled, and that's OK. Signed-off-by: Manuel Pégourié-Gonnard --- tests/scripts/analyze_outcomes.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 81e3ca37f..80b6459cd 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -145,12 +145,10 @@ def analyze_driver_vs_reference(results: Results, outcomes, if component_ref in entry: reference_test_passed = True seen_reference_passing = True - if(reference_test_passed and not driver_test_passed): - if not ignored: - results.error("PASS -> SKIP/FAIL: {}", key) - else: - if ignored: - results.error("uselessly ignored: {}", key) + if reference_test_passed and not driver_test_passed and not ignored: + results.error("PASS -> SKIP/FAIL: {}", key) + if ignored and driver_test_passed: + results.error("uselessly ignored: {}", key) if not seen_reference_passing: results.error("no passing test in reference component: bad outcome file?")