From 8b2df74b125191bf3ae18cdb5ae5f62d15966f7b Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 8 Jul 2022 13:54:57 +0100 Subject: [PATCH 01/47] Add bignum test generation framework Adds python script for generation of bignum test cases, with initial classes for mpi_cmp_mpi test cases. Build scripts are updated to generate test data. Signed-off-by: Werner Lewis --- scripts/make_generated_files.bat | 1 + tests/CMakeLists.txt | 20 +- tests/Makefile | 33 ++- tests/scripts/check-generated-files.sh | 1 + tests/scripts/generate_bignum_tests.py | 269 +++++++++++++++++++++++++ 5 files changed, 313 insertions(+), 11 deletions(-) create mode 100755 tests/scripts/generate_bignum_tests.py diff --git a/scripts/make_generated_files.bat b/scripts/make_generated_files.bat index 662da984c..e9d92758a 100644 --- a/scripts/make_generated_files.bat +++ b/scripts/make_generated_files.bat @@ -10,4 +10,5 @@ perl scripts\generate_features.pl || exit /b 1 python scripts\generate_ssl_debug_helpers.py || exit /b 1 perl scripts\generate_visualc_files.pl || exit /b 1 python scripts\generate_psa_constants.py || exit /b 1 +python tests\scripts\generate_bignum_tests.py || exit /b 1 python tests\scripts\generate_psa_tests.py || exit /b 1 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6049b7473..edb513cd7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,6 +17,17 @@ endif() file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/suites) # Get base names for generated files (starting at "suites/") +execute_process( + COMMAND + ${MBEDTLS_PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py + --list-for-cmake + --directory suites + WORKING_DIRECTORY + ${CMAKE_CURRENT_SOURCE_DIR}/.. + OUTPUT_VARIABLE + bignum_generated_data_files) + execute_process( COMMAND ${MBEDTLS_PYTHON_EXECUTABLE} @@ -26,11 +37,11 @@ execute_process( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. OUTPUT_VARIABLE - base_generated_data_files) + psa_generated_data_files) # Derive generated file paths in the build directory set(generated_data_files "") -foreach(file ${base_generated_data_files}) +foreach(file ${bignum_generated_data_files} ${psa_generated_data_files}) list(APPEND generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file}) endforeach() @@ -44,8 +55,13 @@ if(GEN_FILES) ${MBEDTLS_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py --directory ${CMAKE_CURRENT_BINARY_DIR}/suites + COMMAND + ${MBEDTLS_PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py + --directory ${CMAKE_CURRENT_BINARY_DIR}/suites DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py + ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/crypto_knowledge.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/macro_collector.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/psa_storage.py diff --git a/tests/Makefile b/tests/Makefile index 0d08f845d..e9acca3fe 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -66,23 +66,38 @@ PYTHON ?= $(shell if type python3 >/dev/null 2>/dev/null; then echo python3; els endif .PHONY: generated_files -GENERATED_DATA_FILES := $(patsubst tests/%,%,$(shell \ +GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \ + $(PYTHON) scripts/generate_bignum_tests.py --list || \ + echo FAILED \ +)) +ifeq ($(GENERATED_BIGNUM_DATA_FILES),FAILED) +$(error "$(PYTHON) scripts/generate_bignum_tests.py --list" failed) +endif +GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \ $(PYTHON) scripts/generate_psa_tests.py --list || \ echo FAILED \ )) -ifeq ($(GENERATED_DATA_FILES),FAILED) +ifeq ($(GENERATED_PSA_DATA_FILES),FAILED) $(error "$(PYTHON) scripts/generate_psa_tests.py --list" failed) endif -GENERATED_FILES := $(GENERATED_DATA_FILES) +GENERATED_FILES := $(GENERATED_PSA_DATA_FILES) $(GENERATED_BIGNUM_DATA_FILES) generated_files: $(GENERATED_FILES) -# generate_psa_tests.py spends more time analyzing inputs than generating -# outputs. Its inputs are the same no matter which files are being generated. +# generate_bignum_tests.py and generate_psa_tests.py spend more time analyzing +# inputs than generating outputs. Its inputs are the same no matter which files +# are being generated. # It's rare not to want all the outputs. So always generate all of its outputs. # Use an intermediate phony dependency so that parallel builds don't run # a separate instance of the recipe for each output file. -.SECONDARY: generated_psa_test_data -$(GENERATED_DATA_FILES): generated_psa_test_data +.SECONDARY: generated_bignum_test_data generated_psa_test_data +$(GENERATED_BIGNUM_DATA_FILES): generated_bignum_test_data +generated_bignum_test_data: scripts/generate_bignum_tests.py +generated_bignum_test_data: ../scripts/mbedtls_dev/test_case.py +generated_bignum_test_data: + echo " Gen $(GENERATED_BIGNUM_DATA_FILES)" + $(PYTHON) scripts/generate_bignum_tests.py + +$(GENERATED_PSA_DATA_FILES): generated_psa_test_data generated_psa_test_data: scripts/generate_psa_tests.py generated_psa_test_data: ../scripts/mbedtls_dev/crypto_knowledge.py generated_psa_test_data: ../scripts/mbedtls_dev/macro_collector.py @@ -98,7 +113,7 @@ generated_psa_test_data: ../include/psa/crypto_values.h generated_psa_test_data: ../include/psa/crypto_extra.h generated_psa_test_data: suites/test_suite_psa_crypto_metadata.data generated_psa_test_data: - echo " Gen $(GENERATED_DATA_FILES) ..." + echo " Gen $(GENERATED_PSA_DATA_FILES) ..." $(PYTHON) scripts/generate_psa_tests.py # A test application is built for each suites/test_suite_*.data file. @@ -107,7 +122,7 @@ generated_psa_test_data: DATA_FILES := $(wildcard suites/test_suite_*.data) # Make sure that generated data files are included even if they don't # exist yet when the makefile is parsed. -DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_DATA_FILES)) +DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_FILES)) APPS = $(basename $(subst suites/,,$(DATA_FILES))) # Construct executable name by adding OS specific suffix $(EXEXT). diff --git a/tests/scripts/check-generated-files.sh b/tests/scripts/check-generated-files.sh index 1736f24d2..3006ec7bf 100755 --- a/tests/scripts/check-generated-files.sh +++ b/tests/scripts/check-generated-files.sh @@ -126,4 +126,5 @@ check scripts/generate_ssl_debug_helpers.py library/ssl_debug_helpers_generated. # the step that creates or updates these files. check scripts/generate_visualc_files.pl visualc/VS2010 check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c +check tests/scripts/generate_bignum_tests.py $(tests/scripts/generate_bignum_tests.py --list) check tests/scripts/generate_psa_tests.py $(tests/scripts/generate_psa_tests.py --list) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py new file mode 100755 index 000000000..c6e6a116c --- /dev/null +++ b/tests/scripts/generate_bignum_tests.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Generate test data for bignum functions. + +With no arguments, generate all test data. With non-option arguments, +generate only the specified files. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import itertools +import os +import posixpath +import re +import sys +from typing import Iterable, Iterator, Optional, Tuple, TypeVar + +import scripts_path # pylint: disable=unused-import +from mbedtls_dev import build_tree +from mbedtls_dev import test_case + +T = TypeVar('T') #pylint: disable=invalid-name + +def hex_to_int(val): + return int(val, 16) if val else 0 + +def quote_str(val): + return "\"{}\"".format(val) + + +class BaseTarget: + """Base target for test case generation. + + Attributes: + count: Counter for test class. + desc: Short description of test case. + func: Function which the class generates tests for. + gen_file: File to write generated tests to. + title: Description of the test function/purpose. + """ + count = 0 + desc = None + func = None + gen_file = "" + title = None + + def __init__(self) -> None: + type(self).count += 1 + + @property + def args(self) -> Iterable[str]: + """Create list of arguments for test case.""" + return [] + + @property + def description(self) -> str: + """Create a numbered test description.""" + return "{} #{} {}".format(self.title, self.count, self.desc) + + def create_test_case(self) -> test_case.TestCase: + """Generate test case from the current object.""" + tc = test_case.TestCase() + tc.set_description(self.description) + tc.set_function(self.func) + tc.set_arguments(self.args) + + return tc + + @classmethod + def generate_tests(cls): + """Generate test cases for the target subclasses.""" + for subclass in cls.__subclasses__(): + yield from subclass.generate_tests() + + +class BignumTarget(BaseTarget): + """Target for bignum (mpi) test case generation.""" + gen_file = 'test_suite_mpi.generated' + + +class BignumOperation(BignumTarget): + """Common features for test cases covering bignum operations. + + Attributes: + symb: Symbol used for operation in description. + input_vals: List of values used to generate test case args. + input_cases: List of tuples containing test case inputs. This + can be used to implement specific pairs of inputs. + """ + symb = "" + input_vals = [ + "", "0", "7b", "-7b", + "0000000000000000123", "-0000000000000000123", + "1230000000000000000", "-1230000000000000000" + ] + input_cases = [] + + def __init__(self, val_l: str, val_r: str) -> None: + super().__init__() + + self.arg_l = val_l + self.arg_r = val_r + self.int_l = hex_to_int(val_l) + self.int_r = hex_to_int(val_r) + + @property + def args(self): + return [quote_str(self.arg_l), quote_str(self.arg_r), self.result] + + @property + def description(self): + desc = self.desc if self.desc else "{} {} {}".format( + self.val_desc(self.arg_l), + self.symb, + self.val_desc(self.arg_r) + ) + return "{} #{} {}".format(self.title, self.count, desc) + + @property + def result(self) -> Optional[str]: + return None + + @staticmethod + def val_desc(val) -> str: + """Generate description of the argument val.""" + if val == "": + return "0 (null)" + if val == "0": + return "0 (1 limb)" + + if val[0] == "-": + tmp = "negative" + val = val[1:] + else: + tmp = "positive" + if val[0] == "0": + tmp += " with leading zero limb" + elif len(val) > 10: + tmp = "large " + tmp + return tmp + + @classmethod + def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: + """Generate value pairs.""" + for pair in set( + list(itertools.combinations(cls.input_vals, 2)) + + cls.input_cases + ): + yield pair + + @classmethod + def generate_tests(cls) -> Iterator[test_case.TestCase]: + if cls.func is not None: + # Generate tests for the current class + for l_value, r_value in cls.get_value_pairs(): + cur_op = cls(l_value, r_value) + yield cur_op.create_test_case() + # Once current class completed, check descendants + yield from super().generate_tests() + + +class BignumCmp(BignumOperation): + """Target for bignum comparison test cases.""" + count = 0 + func = "mbedtls_mpi_cmp_mpi" + title = "MPI compare" + input_cases = [ + ("-2", "-3"), + ("-2", "-2"), + ("2b4", "2b5"), + ("2b5", "2b6") + ] + + def __init__(self, val_l, val_r): + super().__init__(val_l, val_r) + self._result = (self.int_l > self.int_r) - (self.int_l < self.int_r) + self.symb = ["<", "==", ">"][self._result + 1] + + @property + def result(self): + return str(self._result) + + +class TestGenerator: + """Generate test data.""" + + def __init__(self, options) -> None: + self.test_suite_directory = self.get_option(options, 'directory', + 'tests/suites') + + @staticmethod + def get_option(options, name: str, default: T) -> T: + value = getattr(options, name, None) + return default if value is None else value + + def filename_for(self, basename: str) -> str: + """The location of the data file with the specified base name.""" + return posixpath.join(self.test_suite_directory, basename + '.data') + + def write_test_data_file(self, basename: str, + test_cases: Iterable[test_case.TestCase]) -> None: + """Write the test cases to a .data file. + + The output file is ``basename + '.data'`` in the test suite directory. + """ + filename = self.filename_for(basename) + test_case.write_data_file(filename, test_cases) + + # Note that targets whose names contain 'test_format' have their content + # validated by `abi_check.py`. + TARGETS = { + subclass.gen_file: subclass.generate_tests for subclass in + BaseTarget.__subclasses__() + } + + def generate_target(self, name: str) -> None: + test_cases = self.TARGETS[name]() + self.write_test_data_file(name, test_cases) + +def main(args): + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--list', action='store_true', + help='List available targets and exit') + parser.add_argument('--list-for-cmake', action='store_true', + help='Print \';\'-separated list of available targets and exit') + parser.add_argument('--directory', metavar='DIR', + help='Output directory (default: tests/suites)') + parser.add_argument('targets', nargs='*', metavar='TARGET', + help='Target file to generate (default: all; "-": none)') + options = parser.parse_args(args) + build_tree.chdir_to_root() + generator = TestGenerator(options) + if options.list: + for name in sorted(generator.TARGETS): + print(generator.filename_for(name)) + return + # List in a cmake list format (i.e. ';'-separated) + if options.list_for_cmake: + print(';'.join(generator.filename_for(name) + for name in sorted(generator.TARGETS)), end='') + return + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_bignum_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.TARGETS) + for target in options.targets: + generator.generate_target(target) + +if __name__ == '__main__': + main(sys.argv[1:]) From 69a92ce497ea40de53b83a9fe60f6b7eaefe6994 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 18 Jul 2022 15:49:43 +0100 Subject: [PATCH 02/47] Add test generation for bignum cmp variant Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index c6e6a116c..36d0d2290 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -194,6 +194,16 @@ class BignumCmp(BignumOperation): return str(self._result) +class BignumCmpAbs(BignumCmp): + """Target for abs comparison variant.""" + count = 0 + func = "mbedtls_mpi_cmp_abs" + title = "MPI compare (abs)" + + def __init__(self, val_l, val_r): + super().__init__(val_l.strip("-"), val_r.strip("-")) + + class TestGenerator: """Generate test data.""" From 86caf85ed23d70bfe64adc3f566fb9dd765b063c Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 18 Jul 2022 17:22:58 +0100 Subject: [PATCH 03/47] Add test case generation for bignum add Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 36d0d2290..e8db99d09 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -204,6 +204,27 @@ class BignumCmpAbs(BignumCmp): super().__init__(val_l.strip("-"), val_r.strip("-")) +class BignumAdd(BignumOperation): + """Target for bignum addition test cases.""" + count = 0 + func = "mbedtls_mpi_add_mpi" + title = "MPI add" + input_cases = list(itertools.combinations( + [ + "1c67967269c6", "9cde3", + "-1c67967269c6", "-9cde3", + ], 2 + )) + + def __init__(self, val_l, val_r): + super().__init__(val_l, val_r) + self.symb = "+" + + @property + def result(self): + return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) + + class TestGenerator: """Generate test data.""" From a51fe2b27e253a4e1cbc8d3b8e112be8d670118d Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 13:35:22 +0100 Subject: [PATCH 04/47] Sort tests when generating cases Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index e8db99d09..76bce5e7b 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -82,7 +82,7 @@ class BaseTarget: @classmethod def generate_tests(cls): """Generate test cases for the target subclasses.""" - for subclass in cls.__subclasses__(): + for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() From b17ca8ad807330ca7b485152a814c86d5a24fa13 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 13:35:53 +0100 Subject: [PATCH 05/47] Remove set() to preserve test case order Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 76bce5e7b..72a10616f 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -155,10 +155,9 @@ class BignumOperation(BignumTarget): @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" - for pair in set( - list(itertools.combinations(cls.input_vals, 2)) + - cls.input_cases - ): + for pair in list( + itertools.combinations(cls.input_vals, 2) + ) + cls.input_cases: yield pair @classmethod From c442f6a3d698770b8d05ec3f8c52120b97f898d4 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 14:13:44 +0100 Subject: [PATCH 06/47] Fix type issues Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 72a10616f..e15261500 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -26,7 +26,7 @@ import os import posixpath import re import sys -from typing import Iterable, Iterator, Optional, Tuple, TypeVar +from typing import Iterable, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import from mbedtls_dev import build_tree @@ -52,16 +52,16 @@ class BaseTarget: title: Description of the test function/purpose. """ count = 0 - desc = None - func = None + desc = "" + func = "" gen_file = "" - title = None + title = "" def __init__(self) -> None: type(self).count += 1 @property - def args(self) -> Iterable[str]: + def args(self) -> List[str]: """Create list of arguments for test case.""" return [] @@ -105,8 +105,8 @@ class BignumOperation(BignumTarget): "", "0", "7b", "-7b", "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" - ] - input_cases = [] + ] # type: List[str] + input_cases = [] # type: List[Tuple[str, ...]] def __init__(self, val_l: str, val_r: str) -> None: super().__init__() From 265e051d06ccd4e1b55b4ff4faa3020154cc7cd8 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 14:45:23 +0100 Subject: [PATCH 07/47] Remove is None from if statement Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index e15261500..299b619e8 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -162,7 +162,7 @@ class BignumOperation(BignumTarget): @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: - if cls.func is not None: + if cls.func: # Generate tests for the current class for l_value, r_value in cls.get_value_pairs(): cur_op = cls(l_value, r_value) From 6a31396a13901f1106942e0d5c5066d71a46455b Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 15:16:50 +0100 Subject: [PATCH 08/47] Fix incorrect indentation Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 299b619e8..61f642b37 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -156,7 +156,7 @@ class BignumOperation(BignumTarget): def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" for pair in list( - itertools.combinations(cls.input_vals, 2) + itertools.combinations(cls.input_vals, 2) ) + cls.input_cases: yield pair From 75ef944da3c3d98bc9b3b6dd10245d6fc4e132a1 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 21 Jul 2022 16:57:22 +0100 Subject: [PATCH 09/47] Fix CMake change failures on Windows Signed-off-by: Werner Lewis --- tests/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index edb513cd7..dcc5de0ff 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,8 +40,9 @@ execute_process( psa_generated_data_files) # Derive generated file paths in the build directory +set(base_generated_data_files ${bignum_generated_data_files} ${psa_generated_data_files}) set(generated_data_files "") -foreach(file ${bignum_generated_data_files} ${psa_generated_data_files}) +foreach(file ${base_generated_data_files}) list(APPEND generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file}) endforeach() From 383461c92fe776cd8e45290fd4a492b2dcf11b48 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 11:29:05 +0100 Subject: [PATCH 10/47] Separate CMake targets for bignum and PSA Signed-off-by: Werner Lewis --- tests/CMakeLists.txt | 65 ++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dcc5de0ff..776d9557d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,7 +26,7 @@ execute_process( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. OUTPUT_VARIABLE - bignum_generated_data_files) + base_bignum_generated_data_files) execute_process( COMMAND @@ -37,32 +37,44 @@ execute_process( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. OUTPUT_VARIABLE - psa_generated_data_files) + base_psa_generated_data_files) # Derive generated file paths in the build directory -set(base_generated_data_files ${bignum_generated_data_files} ${psa_generated_data_files}) -set(generated_data_files "") -foreach(file ${base_generated_data_files}) - list(APPEND generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file}) +set(base_generated_data_files ${base_bignum_generated_data_files} ${base_psa_generated_data_files}) +set(bignum_generated_data_files "") +set(psa_generated_data_files "") +foreach(file ${base_bignum_generated_data_files}) + list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file}) +endforeach() +foreach(file ${base_psa_generated_data_files}) + list(APPEND psa_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file}) endforeach() if(GEN_FILES) add_custom_command( OUTPUT - ${generated_data_files} + ${bignum_generated_data_files} + WORKING_DIRECTORY + ${CMAKE_CURRENT_SOURCE_DIR}/.. + COMMAND + ${MBEDTLS_PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py + --directory ${CMAKE_CURRENT_BINARY_DIR}/suites + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py + ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py + ) + add_custom_command( + OUTPUT + ${psa_generated_data_files} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. COMMAND ${MBEDTLS_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py --directory ${CMAKE_CURRENT_BINARY_DIR}/suites - COMMAND - ${MBEDTLS_PYTHON_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py - --directory ${CMAKE_CURRENT_BINARY_DIR}/suites DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py - ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/crypto_knowledge.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/macro_collector.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/psa_storage.py @@ -82,7 +94,8 @@ endif() # they can cause race conditions in parallel builds. # With this line, only 4 sub-makefiles include the above command, that reduces # the risk of a race. -add_custom_target(test_suite_generated_data DEPENDS ${generated_data_files}) +add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files}) +add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files}) # Test suites caught by SKIP_TEST_SUITES are built but not executed. # "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar" # but not "test_suite_foobar". @@ -99,23 +112,39 @@ function(add_test_suite suite_name) # Get the test names of the tests with generated .data files # from the generated_data_files list in parent scope. - set(generated_data_names "") - foreach(generated_data_file ${generated_data_files}) + set(bignum_generated_data_names "") + set(psa_generated_data_names "") + foreach(generated_data_file ${bignum_generated_data_files}) # Get the plain filename get_filename_component(generated_data_name ${generated_data_file} NAME) # Remove the ".data" extension get_name_without_last_ext(generated_data_name ${generated_data_name}) # Remove leading "test_suite_" string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name) - list(APPEND generated_data_names ${generated_data_name}) + list(APPEND bignum_generated_data_names ${generated_data_name}) + endforeach() + foreach(generated_data_file ${psa_generated_data_files}) + # Get the plain filename + get_filename_component(generated_data_name ${generated_data_file} NAME) + # Remove the ".data" extension + get_name_without_last_ext(generated_data_name ${generated_data_name}) + # Remove leading "test_suite_" + string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name) + list(APPEND psa_generated_data_names ${generated_data_name}) endforeach() - if(";${generated_data_names};" MATCHES ";${data_name};") + if(";${bignum_generated_data_names};" MATCHES ";${data_name};") set(data_file ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data) + set(dependency test_suite_bignum_generated_data) + elseif(";${psa_generated_data_names};" MATCHES ";${data_name};") + set(data_file + ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data) + set(dependency test_suite_psa_generated_data) else() set(data_file ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data) + set(dependency test_suite_bignum_generated_data test_suite_psa_generated_data) endif() add_custom_command( @@ -146,7 +175,7 @@ function(add_test_suite suite_name) ) add_executable(test_suite_${data_name} test_suite_${data_name}.c $) - add_dependencies(test_suite_${data_name} test_suite_generated_data) + add_dependencies(test_suite_${data_name} ${dependency}) target_link_libraries(test_suite_${data_name} ${libs}) # Include test-specific header files from ./include and private header # files (used by some invasive tests) from ../library. Public header From fbb75e3fc5cf5ae2f3580ac29be41001b29eefa4 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 11:30:03 +0100 Subject: [PATCH 11/47] Separate common test generation classes/functions Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 149 +++++++++++++++++++++++++ tests/scripts/generate_bignum_tests.py | 129 ++------------------- tests/scripts/generate_psa_tests.py | 81 ++------------ 3 files changed, 167 insertions(+), 192 deletions(-) create mode 100644 scripts/mbedtls_dev/test_generation.py diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py new file mode 100644 index 000000000..2414f3a4b --- /dev/null +++ b/scripts/mbedtls_dev/test_generation.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Common test generation classes and main function. + +These are used both by generate_psa_tests.py and generate_bignum_tests.py. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import posixpath +import re +from typing import Callable, Dict, Iterable, List, Type, TypeVar + +from mbedtls_dev import build_tree +from mbedtls_dev import test_case + +T = TypeVar('T') #pylint: disable=invalid-name + + +class BaseTarget: + """Base target for test case generation. + + Attributes: + count: Counter for test class. + desc: Short description of test case. + func: Function which the class generates tests for. + gen_file: File to write generated tests to. + title: Description of the test function/purpose. + """ + count = 0 + desc = "" + func = "" + gen_file = "" + title = "" + + def __init__(self) -> None: + type(self).count += 1 + + @property + def args(self) -> List[str]: + """Create list of arguments for test case.""" + return [] + + @property + def description(self) -> str: + """Create a numbered test description.""" + return "{} #{} {}".format(self.title, self.count, self.desc) + + def create_test_case(self) -> test_case.TestCase: + """Generate test case from the current object.""" + tc = test_case.TestCase() + tc.set_description(self.description) + tc.set_function(self.func) + tc.set_arguments(self.args) + + return tc + + @classmethod + def generate_tests(cls): + """Generate test cases for the target subclasses.""" + for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): + yield from subclass.generate_tests() + + +class TestGenerator: + """Generate test data.""" + def __init__(self, options) -> None: + self.test_suite_directory = self.get_option(options, 'directory', + 'tests/suites') + + @staticmethod + def get_option(options, name: str, default: T) -> T: + value = getattr(options, name, None) + return default if value is None else value + + def filename_for(self, basename: str) -> str: + """The location of the data file with the specified base name.""" + return posixpath.join(self.test_suite_directory, basename + '.data') + + def write_test_data_file(self, basename: str, + test_cases: Iterable[test_case.TestCase]) -> None: + """Write the test cases to a .data file. + + The output file is ``basename + '.data'`` in the test suite directory. + """ + filename = self.filename_for(basename) + test_case.write_data_file(filename, test_cases) + + # Note that targets whose names contain 'test_format' have their content + # validated by `abi_check.py`. + TARGETS = {} # type: Dict[str, Callable[..., test_case.TestCase]] + + def generate_target(self, name: str, *target_args) -> None: + """Generate cases and write to data file for a target. + + For target callables which require arguments, override this function + and pass these arguments using super() (see PSATestGenerator). + """ + test_cases = self.TARGETS[name](*target_args) + self.write_test_data_file(name, test_cases) + +def main(args, generator_class: Type[TestGenerator] = TestGenerator): + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--list', action='store_true', + help='List available targets and exit') + parser.add_argument('--list-for-cmake', action='store_true', + help='Print \';\'-separated list of available targets and exit') + parser.add_argument('--directory', metavar='DIR', + help='Output directory (default: tests/suites)') + parser.add_argument('targets', nargs='*', metavar='TARGET', + help='Target file to generate (default: all; "-": none)') + options = parser.parse_args(args) + build_tree.chdir_to_root() + generator = generator_class(options) + if options.list: + for name in sorted(generator.TARGETS): + print(generator.filename_for(name)) + return + # List in a cmake list format (i.e. ';'-separated) + if options.list_for_cmake: + print(';'.join(generator.filename_for(name) + for name in sorted(generator.TARGETS)), end='') + return + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.TARGETS) + for target in options.targets: + generator.generate_target(target) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 61f642b37..f885167cf 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -20,17 +20,13 @@ generate only the specified files. # See the License for the specific language governing permissions and # limitations under the License. -import argparse import itertools -import os -import posixpath -import re import sys -from typing import Iterable, Iterator, List, Optional, Tuple, TypeVar +from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import -from mbedtls_dev import build_tree from mbedtls_dev import test_case +from mbedtls_dev import test_generation T = TypeVar('T') #pylint: disable=invalid-name @@ -41,52 +37,7 @@ def quote_str(val): return "\"{}\"".format(val) -class BaseTarget: - """Base target for test case generation. - - Attributes: - count: Counter for test class. - desc: Short description of test case. - func: Function which the class generates tests for. - gen_file: File to write generated tests to. - title: Description of the test function/purpose. - """ - count = 0 - desc = "" - func = "" - gen_file = "" - title = "" - - def __init__(self) -> None: - type(self).count += 1 - - @property - def args(self) -> List[str]: - """Create list of arguments for test case.""" - return [] - - @property - def description(self) -> str: - """Create a numbered test description.""" - return "{} #{} {}".format(self.title, self.count, self.desc) - - def create_test_case(self) -> test_case.TestCase: - """Generate test case from the current object.""" - tc = test_case.TestCase() - tc.set_description(self.description) - tc.set_function(self.func) - tc.set_arguments(self.args) - - return tc - - @classmethod - def generate_tests(cls): - """Generate test cases for the target subclasses.""" - for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): - yield from subclass.generate_tests() - - -class BignumTarget(BaseTarget): +class BignumTarget(test_generation.BaseTarget): """Target for bignum (mpi) test case generation.""" gen_file = 'test_suite_mpi.generated' @@ -224,76 +175,12 @@ class BignumAdd(BignumOperation): return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) -class TestGenerator: - """Generate test data.""" - - def __init__(self, options) -> None: - self.test_suite_directory = self.get_option(options, 'directory', - 'tests/suites') - - @staticmethod - def get_option(options, name: str, default: T) -> T: - value = getattr(options, name, None) - return default if value is None else value - - def filename_for(self, basename: str) -> str: - """The location of the data file with the specified base name.""" - return posixpath.join(self.test_suite_directory, basename + '.data') - - def write_test_data_file(self, basename: str, - test_cases: Iterable[test_case.TestCase]) -> None: - """Write the test cases to a .data file. - - The output file is ``basename + '.data'`` in the test suite directory. - """ - filename = self.filename_for(basename) - test_case.write_data_file(filename, test_cases) - - # Note that targets whose names contain 'test_format' have their content - # validated by `abi_check.py`. +class BignumTestGenerator(test_generation.TestGenerator): + """Test generator subclass including bignum targets.""" TARGETS = { subclass.gen_file: subclass.generate_tests for subclass in - BaseTarget.__subclasses__() - } - - def generate_target(self, name: str) -> None: - test_cases = self.TARGETS[name]() - self.write_test_data_file(name, test_cases) - -def main(args): - """Command line entry point.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--list', action='store_true', - help='List available targets and exit') - parser.add_argument('--list-for-cmake', action='store_true', - help='Print \';\'-separated list of available targets and exit') - parser.add_argument('--directory', metavar='DIR', - help='Output directory (default: tests/suites)') - parser.add_argument('targets', nargs='*', metavar='TARGET', - help='Target file to generate (default: all; "-": none)') - options = parser.parse_args(args) - build_tree.chdir_to_root() - generator = TestGenerator(options) - if options.list: - for name in sorted(generator.TARGETS): - print(generator.filename_for(name)) - return - # List in a cmake list format (i.e. ';'-separated) - if options.list_for_cmake: - print(';'.join(generator.filename_for(name) - for name in sorted(generator.TARGETS)), end='') - return - if options.targets: - # Allow "-" as a special case so you can run - # ``generate_bignum_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] - else: - options.targets = sorted(generator.TARGETS) - for target in options.targets: - generator.generate_target(target) + test_generation.BaseTarget.__subclasses__() + } # type: Dict[str, Callable[[], test_case.TestCase]] if __name__ == '__main__': - main(sys.argv[1:]) + test_generation.main(sys.argv[1:], BignumTestGenerator) diff --git a/tests/scripts/generate_psa_tests.py b/tests/scripts/generate_psa_tests.py index 3d23edda6..9f32655ae 100755 --- a/tests/scripts/generate_psa_tests.py +++ b/tests/scripts/generate_psa_tests.py @@ -20,22 +20,17 @@ generate only the specified files. # See the License for the specific language governing permissions and # limitations under the License. -import argparse import enum -import os -import posixpath import re import sys -from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional, TypeVar +from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional import scripts_path # pylint: disable=unused-import -from mbedtls_dev import build_tree from mbedtls_dev import crypto_knowledge from mbedtls_dev import macro_collector from mbedtls_dev import psa_storage from mbedtls_dev import test_case - -T = TypeVar('T') #pylint: disable=invalid-name +from mbedtls_dev import test_generation def psa_want_symbol(name: str) -> str: @@ -897,32 +892,8 @@ class StorageFormatV0(StorageFormat): yield from super().generate_all_keys() yield from self.all_keys_for_implicit_usage() -class TestGenerator: - """Generate test data.""" - - def __init__(self, options) -> None: - self.test_suite_directory = self.get_option(options, 'directory', - 'tests/suites') - self.info = Information() - - @staticmethod - def get_option(options, name: str, default: T) -> T: - value = getattr(options, name, None) - return default if value is None else value - - def filename_for(self, basename: str) -> str: - """The location of the data file with the specified base name.""" - return posixpath.join(self.test_suite_directory, basename + '.data') - - def write_test_data_file(self, basename: str, - test_cases: Iterable[test_case.TestCase]) -> None: - """Write the test cases to a .data file. - - The output file is ``basename + '.data'`` in the test suite directory. - """ - filename = self.filename_for(basename) - test_case.write_data_file(filename, test_cases) - +class PSATestGenerator(test_generation.TestGenerator): + """Test generator subclass including PSA targets and info.""" # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. TARGETS = { @@ -938,44 +909,12 @@ class TestGenerator: lambda info: StorageFormatV0(info).all_test_cases(), } #type: Dict[str, Callable[[Information], Iterable[test_case.TestCase]]] - def generate_target(self, name: str) -> None: - test_cases = self.TARGETS[name](self.info) - self.write_test_data_file(name, test_cases) + def __init__(self, options): + super().__init__(options) + self.info = Information() -def main(args): - """Command line entry point.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--list', action='store_true', - help='List available targets and exit') - parser.add_argument('--list-for-cmake', action='store_true', - help='Print \';\'-separated list of available targets and exit') - parser.add_argument('--directory', metavar='DIR', - help='Output directory (default: tests/suites)') - parser.add_argument('targets', nargs='*', metavar='TARGET', - help='Target file to generate (default: all; "-": none)') - options = parser.parse_args(args) - build_tree.chdir_to_root() - generator = TestGenerator(options) - if options.list: - for name in sorted(generator.TARGETS): - print(generator.filename_for(name)) - return - # List in a cmake list format (i.e. ';'-separated) - if options.list_for_cmake: - print(';'.join(generator.filename_for(name) - for name in sorted(generator.TARGETS)), end='') - return - if options.targets: - # Allow "-" as a special case so you can run - # ``generate_psa_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] - else: - options.targets = sorted(generator.TARGETS) - for target in options.targets: - generator.generate_target(target) + def generate_target(self, name: str, *target_args) -> None: + super().generate_target(name, self.info) if __name__ == '__main__': - main(sys.argv[1:]) + test_generation.main(sys.argv[1:], PSATestGenerator) From 55e638ca57492204a27601b5b30f9af7a075a9d8 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 14:21:53 +0100 Subject: [PATCH 12/47] Remove abbreviations and clarify attributes Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 34 +++++++-------- tests/scripts/generate_bignum_tests.py | 59 ++++++++++++-------------- 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 2414f3a4b..bb70b9c72 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -35,37 +35,37 @@ class BaseTarget: """Base target for test case generation. Attributes: - count: Counter for test class. - desc: Short description of test case. - func: Function which the class generates tests for. - gen_file: File to write generated tests to. - title: Description of the test function/purpose. + count: Counter for test cases from this class. + case_description: Short description of the test case. This may be + automatically generated using the class, or manually set. + target_basename: Basename of file to write generated tests to. This + should be specified in a child class of BaseTarget. + test_function: Test function which the class generates cases for. + test_name: A common name or description of the test function. This can + be the function's name, or a short summary of its purpose. """ count = 0 - desc = "" - func = "" - gen_file = "" - title = "" + case_description = "" + target_basename = "" + test_function = "" + test_name = "" def __init__(self) -> None: type(self).count += 1 - @property - def args(self) -> List[str]: - """Create list of arguments for test case.""" + def arguments(self) -> List[str]: return [] - @property def description(self) -> str: """Create a numbered test description.""" - return "{} #{} {}".format(self.title, self.count, self.desc) + return "{} #{} {}".format(self.test_name, self.count, self.case_description) def create_test_case(self) -> test_case.TestCase: """Generate test case from the current object.""" tc = test_case.TestCase() - tc.set_description(self.description) - tc.set_function(self.func) - tc.set_arguments(self.args) + tc.set_description(self.description()) + tc.set_function(self.test_function) + tc.set_arguments(self.arguments()) return tc diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index f885167cf..fbccb8a9f 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -39,20 +39,20 @@ def quote_str(val): class BignumTarget(test_generation.BaseTarget): """Target for bignum (mpi) test case generation.""" - gen_file = 'test_suite_mpi.generated' + target_basename = 'test_suite_mpi.generated' class BignumOperation(BignumTarget): """Common features for test cases covering bignum operations. Attributes: - symb: Symbol used for operation in description. - input_vals: List of values used to generate test case args. - input_cases: List of tuples containing test case inputs. This + symbol: Symbol used for operation in description. + input_values: List of values to use as test case inputs. + input_cases: List of tuples containing pairs of test case inputs. This can be used to implement specific pairs of inputs. """ - symb = "" - input_vals = [ + symbol = "" + input_values = [ "", "0", "7b", "-7b", "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" @@ -67,26 +67,23 @@ class BignumOperation(BignumTarget): self.int_l = hex_to_int(val_l) self.int_r = hex_to_int(val_r) - @property - def args(self): - return [quote_str(self.arg_l), quote_str(self.arg_r), self.result] + def arguments(self): + return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] - @property def description(self): - desc = self.desc if self.desc else "{} {} {}".format( - self.val_desc(self.arg_l), - self.symb, - self.val_desc(self.arg_r) - ) - return "{} #{} {}".format(self.title, self.count, desc) + if not self.case_description: + self.case_description = "{} {} {}".format( + self.value_description(self.arg_l), + self.symbol, + self.value_description(self.arg_r) + ) + return super().description() - @property def result(self) -> Optional[str]: return None @staticmethod - def val_desc(val) -> str: - """Generate description of the argument val.""" + def value_description(val) -> str: if val == "": return "0 (null)" if val == "0": @@ -107,13 +104,13 @@ class BignumOperation(BignumTarget): def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" for pair in list( - itertools.combinations(cls.input_vals, 2) + itertools.combinations(cls.input_values, 2) ) + cls.input_cases: yield pair @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: - if cls.func: + if cls.test_function: # Generate tests for the current class for l_value, r_value in cls.get_value_pairs(): cur_op = cls(l_value, r_value) @@ -125,8 +122,8 @@ class BignumOperation(BignumTarget): class BignumCmp(BignumOperation): """Target for bignum comparison test cases.""" count = 0 - func = "mbedtls_mpi_cmp_mpi" - title = "MPI compare" + test_function = "mbedtls_mpi_cmp_mpi" + test_name = "MPI compare" input_cases = [ ("-2", "-3"), ("-2", "-2"), @@ -137,9 +134,8 @@ class BignumCmp(BignumOperation): def __init__(self, val_l, val_r): super().__init__(val_l, val_r) self._result = (self.int_l > self.int_r) - (self.int_l < self.int_r) - self.symb = ["<", "==", ">"][self._result + 1] + self.symbol = ["<", "==", ">"][self._result + 1] - @property def result(self): return str(self._result) @@ -147,8 +143,8 @@ class BignumCmp(BignumOperation): class BignumCmpAbs(BignumCmp): """Target for abs comparison variant.""" count = 0 - func = "mbedtls_mpi_cmp_abs" - title = "MPI compare (abs)" + test_function = "mbedtls_mpi_cmp_abs" + test_name = "MPI compare (abs)" def __init__(self, val_l, val_r): super().__init__(val_l.strip("-"), val_r.strip("-")) @@ -157,8 +153,8 @@ class BignumCmpAbs(BignumCmp): class BignumAdd(BignumOperation): """Target for bignum addition test cases.""" count = 0 - func = "mbedtls_mpi_add_mpi" - title = "MPI add" + test_function = "mbedtls_mpi_add_mpi" + test_name = "MPI add" input_cases = list(itertools.combinations( [ "1c67967269c6", "9cde3", @@ -168,9 +164,8 @@ class BignumAdd(BignumOperation): def __init__(self, val_l, val_r): super().__init__(val_l, val_r) - self.symb = "+" + self.symbol = "+" - @property def result(self): return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) @@ -178,7 +173,7 @@ class BignumAdd(BignumOperation): class BignumTestGenerator(test_generation.TestGenerator): """Test generator subclass including bignum targets.""" TARGETS = { - subclass.gen_file: subclass.generate_tests for subclass in + subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() } # type: Dict[str, Callable[[], test_case.TestCase]] From 92c876aaa9258928123469593b36097c9834b937 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 16:07:19 +0100 Subject: [PATCH 13/47] Remove unneeded list concatenation Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index fbccb8a9f..757a80a44 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -103,10 +103,8 @@ class BignumOperation(BignumTarget): @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" - for pair in list( - itertools.combinations(cls.input_values, 2) - ) + cls.input_cases: - yield pair + yield from itertools.combinations(cls.input_values, 2) + yield from cls.input_cases @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: From 6c70d745d17f7b67a4e8d010e770e3414bdaa55a Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 16:37:44 +0100 Subject: [PATCH 14/47] Convert bools to int before arithmetic Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 757a80a44..471fd7724 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -131,7 +131,7 @@ class BignumCmp(BignumOperation): def __init__(self, val_l, val_r): super().__init__(val_l, val_r) - self._result = (self.int_l > self.int_r) - (self.int_l < self.int_r) + self._result = int(self.int_l > self.int_r) - int(self.int_l < self.int_r) self.symbol = ["<", "==", ">"][self._result + 1] def result(self): From 169034ae63541649631b0ac851a93bf234b9250e Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 16:07:37 +0100 Subject: [PATCH 15/47] Add details to docstrings Clarification is added to docstrings, mostly in abstract classes. Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 38 +++++++++++++-- tests/scripts/generate_bignum_tests.py | 65 +++++++++++++++++++++++--- 2 files changed, 93 insertions(+), 10 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index bb70b9c72..712c7996b 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -23,6 +23,8 @@ import argparse import os import posixpath import re + +from abc import abstractmethod from typing import Callable, Dict, Iterable, List, Type, TypeVar from mbedtls_dev import build_tree @@ -53,15 +55,34 @@ class BaseTarget: def __init__(self) -> None: type(self).count += 1 + @abstractmethod def arguments(self) -> List[str]: - return [] + """Get the list of arguments for the test case. + + Override this method to provide the list of arguments required for + generating the test_function. + + Returns: + List of arguments required for the test function. + """ + pass def description(self) -> str: - """Create a numbered test description.""" + """Create a test description. + + Creates a description of the test case, including a name for the test + function, and describing the specific test case. This should inform a + reader of the purpose of the case. The case description may be + generated in the class, or provided manually as needed. + + Returns: + Description for the test case. + """ return "{} #{} {}".format(self.test_name, self.count, self.case_description) + def create_test_case(self) -> test_case.TestCase: - """Generate test case from the current object.""" + """Generate TestCase from the current object.""" tc = test_case.TestCase() tc.set_description(self.description()) tc.set_function(self.test_function) @@ -71,7 +92,16 @@ class BaseTarget: @classmethod def generate_tests(cls): - """Generate test cases for the target subclasses.""" + """Generate test cases for the target subclasses. + + Classes will iterate over its subclasses, calling this method in each. + In abstract classes, no further changes are needed, as there is no + function to generate tests for. + In classes which do implement a test function, this should be overrided + and a means to use `create_test_case()` should be added. In most cases + the subclasses can still be iterated over, as either the class will + have none, or it may continue. + """ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 471fd7724..7a8ebd1d8 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -3,6 +3,31 @@ With no arguments, generate all test data. With non-option arguments, generate only the specified files. + +Class structure: + +Target classes are directly derived from test_generation.BaseTarget, +representing a target file. These indicate where test cases will be written +to in classes derived from the Target. Multiple Target classes must not +represent the same target_basename. + +Each subclass derived from a Target can either be: + - A concrete class, representing a test function, which generates test cases. + - An abstract class containing shared methods and attributes, not associated + with a test function. An example is BignumOperation, which provides common + features used in binary bignum operations. + + +Adding test generation for a function: + +A subclass representing the test function should be added, deriving from a +Target class or a descendant. This subclass must set/implement the following: + - test_function: the function name from the associated .function file. + - arguments(): generation of the arguments required for the test_function. + - generate_function_test(): generation of the test cases for the function. + +Additional details and other attributes/methods are given in the documentation +of BaseTarget in test_generation.py. """ # Copyright The Mbed TLS Contributors @@ -22,6 +47,8 @@ generate only the specified files. import itertools import sys + +from abc import abstractmethod from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import @@ -43,11 +70,16 @@ class BignumTarget(test_generation.BaseTarget): class BignumOperation(BignumTarget): - """Common features for test cases covering bignum operations. + """Common features for test cases covering binary bignum operations. + + This adds functionality common in binary operation tests. This includes + generation of case descriptions, using descriptions of values and symbols + to represent the operation or result. Attributes: - symbol: Symbol used for operation in description. - input_values: List of values to use as test case inputs. + symbol: Symbol used for the operation in case description. + input_values: List of values to use as test case inputs. These are + combined to produce pairs of values. input_cases: List of tuples containing pairs of test case inputs. This can be used to implement specific pairs of inputs. """ @@ -71,6 +103,12 @@ class BignumOperation(BignumTarget): return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] def description(self): + """Generate a description for the test case. + + If not set, case_description uses the form A `symbol` B, where symbol + is used to represent the operation. Descriptions of each value are + generated to provide some context to the test case. + """ if not self.case_description: self.case_description = "{} {} {}".format( self.value_description(self.arg_l), @@ -79,11 +117,22 @@ class BignumOperation(BignumTarget): ) return super().description() + @abstractmethod def result(self) -> Optional[str]: - return None + """Get the result of the operation. + + This may be calculated during initialization and stored as `_result`, + or calculated when the method is called. + """ + pass @staticmethod def value_description(val) -> str: + """Generate a description of the argument val. + + This produces a simple description of the value, which are used in test + case naming, to avoid most generated cases only being numbered. + """ if val == "": return "0 (null)" if val == "0": @@ -102,7 +151,11 @@ class BignumOperation(BignumTarget): @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: - """Generate value pairs.""" + """Generator for pairs of inputs. + + Combinations are first generated from all input values, and then + specific cases provided. + """ yield from itertools.combinations(cls.input_values, 2) yield from cls.input_cases @@ -139,7 +192,7 @@ class BignumCmp(BignumOperation): class BignumCmpAbs(BignumCmp): - """Target for abs comparison variant.""" + """Target for bignum comparison, absolute variant.""" count = 0 test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" From 699e126942fef15142dfc6a95604360566e19dd2 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 12:18:25 +0100 Subject: [PATCH 16/47] Use ABCMeta for abstract classes Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 15 +++++++-------- tests/scripts/generate_bignum_tests.py | 10 +++++----- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 712c7996b..b825df07b 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -24,7 +24,7 @@ import os import posixpath import re -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from typing import Callable, Dict, Iterable, List, Type, TypeVar from mbedtls_dev import build_tree @@ -33,7 +33,7 @@ from mbedtls_dev import test_case T = TypeVar('T') #pylint: disable=invalid-name -class BaseTarget: +class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. Attributes: @@ -94,13 +94,12 @@ class BaseTarget: def generate_tests(cls): """Generate test cases for the target subclasses. - Classes will iterate over its subclasses, calling this method in each. - In abstract classes, no further changes are needed, as there is no + During generation, each class will iterate over any subclasses, calling + this method in each. + In abstract classes, no tests will be generated, as there is no function to generate tests for. - In classes which do implement a test function, this should be overrided - and a means to use `create_test_case()` should be added. In most cases - the subclasses can still be iterated over, as either the class will - have none, or it may continue. + In classes which do implement a test function, this should be overridden + and a means to use `create_test_case()` should be added. """ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 7a8ebd1d8..3f556ce29 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -48,7 +48,7 @@ of BaseTarget in test_generation.py. import itertools import sys -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import @@ -64,12 +64,12 @@ def quote_str(val): return "\"{}\"".format(val) -class BignumTarget(test_generation.BaseTarget): +class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): """Target for bignum (mpi) test case generation.""" target_basename = 'test_suite_mpi.generated' -class BignumOperation(BignumTarget): +class BignumOperation(BignumTarget, metaclass=ABCMeta): """Common features for test cases covering binary bignum operations. This adds functionality common in binary operation tests. This includes @@ -118,7 +118,7 @@ class BignumOperation(BignumTarget): return super().description() @abstractmethod - def result(self) -> Optional[str]: + def result(self) -> str: """Get the result of the operation. This may be calculated during initialization and stored as `_result`, @@ -131,7 +131,7 @@ class BignumOperation(BignumTarget): """Generate a description of the argument val. This produces a simple description of the value, which are used in test - case naming, to avoid most generated cases only being numbered. + case naming, to add context to the test cases. """ if val == "": return "0 (null)" From 2b527a394dd851905bb73a6ccfd810fbbd7c6d77 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 12:42:00 +0100 Subject: [PATCH 17/47] Split generate_tests to reduce code complexity Previous implementation mixed the test case generation and the recursive generation calls together. A separate method is added to generate test cases for the current class' test function. This reduces the need to override generate_tests(). Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 33 +++++++++++++++++++------- tests/scripts/generate_bignum_tests.py | 12 ++++------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index b825df07b..aeb551d05 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -25,7 +25,7 @@ import posixpath import re from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterable, List, Type, TypeVar +from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar from mbedtls_dev import build_tree from mbedtls_dev import test_case @@ -91,16 +91,31 @@ class BaseTarget(metaclass=ABCMeta): return tc @classmethod - def generate_tests(cls): - """Generate test cases for the target subclasses. + @abstractmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + """Generate test cases for the test function. - During generation, each class will iterate over any subclasses, calling - this method in each. - In abstract classes, no tests will be generated, as there is no - function to generate tests for. - In classes which do implement a test function, this should be overridden - and a means to use `create_test_case()` should be added. + This will be called in classes where `test_function` is set. + Implementations should yield TestCase objects, by creating instances + of the class with appropriate input data, and then calling + `create_test_case()` on each. """ + pass + + @classmethod + def generate_tests(cls) -> Iterator[test_case.TestCase]: + """Generate test cases for the class and its subclasses. + + In classes with `test_function` set, `generate_function_tests()` is + used to generate test cases first. + In all classes, this method will iterate over its subclasses, and + yield from `generate_tests()` in each. + + Calling this method on a class X will yield test cases from all classes + derived from X. + """ + if cls.test_function: + yield from cls.generate_function_tests() for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 3f556ce29..1f6448528 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -160,14 +160,10 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): yield from cls.input_cases @classmethod - def generate_tests(cls) -> Iterator[test_case.TestCase]: - if cls.test_function: - # Generate tests for the current class - for l_value, r_value in cls.get_value_pairs(): - cur_op = cls(l_value, r_value) - yield cur_op.create_test_case() - # Once current class completed, check descendants - yield from super().generate_tests() + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for l_value, r_value in cls.get_value_pairs(): + cur_op = cls(l_value, r_value) + yield cur_op.create_test_case() class BignumCmp(BignumOperation): From cfd4768df2e510127235848d4a8cdc5010811bef Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:04:07 +0100 Subject: [PATCH 18/47] Use __new__() for case counting Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 5 +++-- tests/scripts/generate_bignum_tests.py | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index aeb551d05..f1e085d4e 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -52,8 +52,9 @@ class BaseTarget(metaclass=ABCMeta): test_function = "" test_name = "" - def __init__(self) -> None: - type(self).count += 1 + def __new__(cls, *args, **kwargs): + cls.count += 1 + return super().__new__(cls) @abstractmethod def arguments(self) -> List[str]: diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 1f6448528..9551e2186 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -92,8 +92,6 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): input_cases = [] # type: List[Tuple[str, ...]] def __init__(self, val_l: str, val_r: str) -> None: - super().__init__() - self.arg_l = val_l self.arg_r = val_r self.int_l = hex_to_int(val_l) From d03d2a3a917e25569443bbd49189374fff76ea71 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:20:29 +0100 Subject: [PATCH 19/47] Remove trailing whitespace in description Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index f1e085d4e..23d9c7e55 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -79,7 +79,9 @@ class BaseTarget(metaclass=ABCMeta): Returns: Description for the test case. """ - return "{} #{} {}".format(self.test_name, self.count, self.case_description) + return "{} #{} {}".format( + self.test_name, self.count, self.case_description + ).strip() def create_test_case(self) -> test_case.TestCase: From 6300b4f7e077bebb843de9ec75231739d4364ee2 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:46:22 +0100 Subject: [PATCH 20/47] Add missing typing Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 9551e2186..016e03771 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -49,7 +49,7 @@ import itertools import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar +from typing import Callable, Dict, Iterator, List, Tuple, TypeVar import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -57,10 +57,10 @@ from mbedtls_dev import test_generation T = TypeVar('T') #pylint: disable=invalid-name -def hex_to_int(val): +def hex_to_int(val: str) -> int: return int(val, 16) if val else 0 -def quote_str(val): +def quote_str(val) -> str: return "\"{}\"".format(val) @@ -89,7 +89,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" ] # type: List[str] - input_cases = [] # type: List[Tuple[str, ...]] + input_cases = [] # type: List[Tuple[str, str]] def __init__(self, val_l: str, val_r: str) -> None: self.arg_l = val_l @@ -97,10 +97,10 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): self.int_l = hex_to_int(val_l) self.int_r = hex_to_int(val_r) - def arguments(self): + def arguments(self) -> List[str]: return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] - def description(self): + def description(self) -> str: """Generate a description for the test case. If not set, case_description uses the form A `symbol` B, where symbol @@ -148,7 +148,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): return tmp @classmethod - def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: + def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: """Generator for pairs of inputs. Combinations are first generated from all input values, and then @@ -176,12 +176,12 @@ class BignumCmp(BignumOperation): ("2b5", "2b6") ] - def __init__(self, val_l, val_r): + def __init__(self, val_l, val_r) -> None: super().__init__(val_l, val_r) self._result = int(self.int_l > self.int_r) - int(self.int_l < self.int_r) self.symbol = ["<", "==", ">"][self._result + 1] - def result(self): + def result(self) -> str: return str(self._result) @@ -191,7 +191,7 @@ class BignumCmpAbs(BignumCmp): test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" - def __init__(self, val_l, val_r): + def __init__(self, val_l, val_r) -> None: super().__init__(val_l.strip("-"), val_r.strip("-")) @@ -207,11 +207,11 @@ class BignumAdd(BignumOperation): ], 2 )) - def __init__(self, val_l, val_r): + def __init__(self, val_l, val_r) -> None: super().__init__(val_l, val_r) self.symbol = "+" - def result(self): + def result(self) -> str: return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) From 9990b30568b4c7ee405bf2fe4d9dc4f1d685b2cd Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 18:03:30 +0100 Subject: [PATCH 21/47] Use typing casts for fixed-width tuples Enforces fixed-width tuple types where mypy does not recognize. Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 016e03771..a2a9d0674 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -49,7 +49,7 @@ import itertools import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterator, List, Tuple, TypeVar +from typing import Callable, Dict, Iterator, List, Tuple, TypeVar, cast import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -89,7 +89,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" ] # type: List[str] - input_cases = [] # type: List[Tuple[str, str]] + input_cases = cast(List[Tuple[str, str]], []) # type: List[Tuple[str, str]] def __init__(self, val_l: str, val_r: str) -> None: self.arg_l = val_l @@ -154,7 +154,10 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): Combinations are first generated from all input values, and then specific cases provided. """ - yield from itertools.combinations(cls.input_values, 2) + yield from cast( + Iterator[Tuple[str, str]], + itertools.combinations(cls.input_values, 2) + ) yield from cls.input_cases @classmethod @@ -200,12 +203,15 @@ class BignumAdd(BignumOperation): count = 0 test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" - input_cases = list(itertools.combinations( + input_cases = cast( + List[Tuple[str, str]], + list(itertools.combinations( [ "1c67967269c6", "9cde3", "-1c67967269c6", "-9cde3", ], 2 - )) + )) + ) def __init__(self, val_l, val_r) -> None: super().__init__(val_l, val_r) From a195ce73f5a17ccf61ad9de8473bf1bb30751c31 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 18:09:10 +0100 Subject: [PATCH 22/47] Disable pylint unused arg in __new__ Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 23d9c7e55..652b9a1f8 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -53,6 +53,7 @@ class BaseTarget(metaclass=ABCMeta): test_name = "" def __new__(cls, *args, **kwargs): + # pylint: disable=unused-argument cls.count += 1 return super().__new__(cls) From 6d654c6491eb44cd0e221e2ce3aa72a6173bf9ff Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 09:56:51 +0100 Subject: [PATCH 23/47] Raise NotImplementedError in abstract methods Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 ++-- tests/scripts/generate_bignum_tests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 652b9a1f8..a90547349 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -67,7 +67,7 @@ class BaseTarget(metaclass=ABCMeta): Returns: List of arguments required for the test function. """ - pass + raise NotImplementedError def description(self) -> str: """Create a test description. @@ -104,7 +104,7 @@ class BaseTarget(metaclass=ABCMeta): of the class with appropriate input data, and then calling `create_test_case()` on each. """ - pass + raise NotImplementedError @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index a2a9d0674..aa7e131a1 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -122,7 +122,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): This may be calculated during initialization and stored as `_result`, or calculated when the method is called. """ - pass + raise NotImplementedError @staticmethod def value_description(val) -> str: From e3ad22ecf256a6d32bb4e417da01b611ffb9ed94 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 10:02:06 +0100 Subject: [PATCH 24/47] Fix TARGET types and code style Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 3 +-- tests/scripts/generate_bignum_tests.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index a90547349..11c085f6b 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Common test generation classes and main function. These are used both by generate_psa_tests.py and generate_bignum_tests.py. @@ -150,7 +149,7 @@ class TestGenerator: # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = {} # type: Dict[str, Callable[..., test_case.TestCase]] + TARGETS = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] def generate_target(self, name: str, *target_args) -> None: """Generate cases and write to data file for a target. diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index aa7e131a1..c57f197a2 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -49,7 +49,7 @@ import itertools import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterator, List, Tuple, TypeVar, cast +from typing import Callable, Dict, Iterable, Iterator, List, Tuple, TypeVar, cast import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -206,10 +206,10 @@ class BignumAdd(BignumOperation): input_cases = cast( List[Tuple[str, str]], list(itertools.combinations( - [ - "1c67967269c6", "9cde3", - "-1c67967269c6", "-9cde3", - ], 2 + [ + "1c67967269c6", "9cde3", + "-1c67967269c6", "-9cde3", + ], 2 )) ) @@ -226,7 +226,7 @@ class BignumTestGenerator(test_generation.TestGenerator): TARGETS = { subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() - } # type: Dict[str, Callable[[], test_case.TestCase]] + } # type: Dict[str, Callable[[], Iterable[test_case.TestCase]]] if __name__ == '__main__': test_generation.main(sys.argv[1:], BignumTestGenerator) From a16b617fe9844ccabde8ee80e1245b292a418f39 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 11:17:35 +0100 Subject: [PATCH 25/47] Disable abstract check in pylint Version of pylint used in CI does not recognize abstract subclasses of BaseTarget, so disable warning in these abstract classes. Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index c57f197a2..2443f659b 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -65,6 +65,7 @@ def quote_str(val) -> str: class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): + #pylint: disable=abstract-method """Target for bignum (mpi) test case generation.""" target_basename = 'test_suite_mpi.generated' From f156c43702bec949c80a56a993a0d5009e39ab17 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 11:30:17 +0100 Subject: [PATCH 26/47] Use argparser default for directory Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 11c085f6b..4803c24b6 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -126,13 +126,7 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: """Generate test data.""" def __init__(self, options) -> None: - self.test_suite_directory = self.get_option(options, 'directory', - 'tests/suites') - - @staticmethod - def get_option(options, name: str, default: T) -> T: - value = getattr(options, name, None) - return default if value is None else value + self.test_suite_directory = getattr(options, 'directory') def filename_for(self, basename: str) -> str: """The location of the data file with the specified base name.""" @@ -167,7 +161,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): help='List available targets and exit') parser.add_argument('--list-for-cmake', action='store_true', help='Print \';\'-separated list of available targets and exit') - parser.add_argument('--directory', metavar='DIR', + parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', help='Target file to generate (default: all; "-": none)') From 6ef5436f3cf5c2efb926448f6eb52ddb301bff6e Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 12:29:46 +0100 Subject: [PATCH 27/47] Clarify documentation Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 26 ++++++++------- tests/scripts/generate_bignum_tests.py | 44 +++++++++++++++----------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 4803c24b6..9e004a69f 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -35,6 +35,8 @@ T = TypeVar('T') #pylint: disable=invalid-name class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. + This should be derived from for file Targets. + Attributes: count: Counter for test cases from this class. case_description: Short description of the test case. This may be @@ -43,7 +45,8 @@ class BaseTarget(metaclass=ABCMeta): should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. test_name: A common name or description of the test function. This can - be the function's name, or a short summary of its purpose. + be `test_function`, a clearer equivalent, or a short summary of the + test function's purpose. """ count = 0 case_description = "" @@ -61,7 +64,7 @@ class BaseTarget(metaclass=ABCMeta): """Get the list of arguments for the test case. Override this method to provide the list of arguments required for - generating the test_function. + the `test_function`. Returns: List of arguments required for the test function. @@ -69,12 +72,12 @@ class BaseTarget(metaclass=ABCMeta): raise NotImplementedError def description(self) -> str: - """Create a test description. + """Create a test case description. Creates a description of the test case, including a name for the test - function, and describing the specific test case. This should inform a - reader of the purpose of the case. The case description may be - generated in the class, or provided manually as needed. + function, a case number, and a description the specific test case. + This should inform a reader what is being tested, and provide context + for the test case. Returns: Description for the test case. @@ -85,7 +88,7 @@ class BaseTarget(metaclass=ABCMeta): def create_test_case(self) -> test_case.TestCase: - """Generate TestCase from the current object.""" + """Generate TestCase from the instance.""" tc = test_case.TestCase() tc.set_description(self.description()) tc.set_function(self.test_function) @@ -96,7 +99,7 @@ class BaseTarget(metaclass=ABCMeta): @classmethod @abstractmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - """Generate test cases for the test function. + """Generate test cases for the class test function. This will be called in classes where `test_function` is set. Implementations should yield TestCase objects, by creating instances @@ -111,11 +114,10 @@ class BaseTarget(metaclass=ABCMeta): In classes with `test_function` set, `generate_function_tests()` is used to generate test cases first. - In all classes, this method will iterate over its subclasses, and - yield from `generate_tests()` in each. - Calling this method on a class X will yield test cases from all classes - derived from X. + In all classes, this method will iterate over its subclasses, and + yield from `generate_tests()` in each. Calling this method on a class X + will yield test cases from all classes derived from X. """ if cls.test_function: yield from cls.generate_function_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 2443f659b..4486d4958 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -6,25 +6,33 @@ generate only the specified files. Class structure: -Target classes are directly derived from test_generation.BaseTarget, -representing a target file. These indicate where test cases will be written -to in classes derived from the Target. Multiple Target classes must not -represent the same target_basename. +Child classes of test_generation.BaseTarget (file Targets) represent a target +file. These indicate where test cases will be written to, for all subclasses of +this Target. Multiple Target classes should not reuse a `target_basename`. -Each subclass derived from a Target can either be: +Each subclass derived from a file Target can either be: - A concrete class, representing a test function, which generates test cases. - An abstract class containing shared methods and attributes, not associated - with a test function. An example is BignumOperation, which provides common - features used in binary bignum operations. + with a test function. An example is BignumOperation, which provides + common features used for bignum binary operations. + +Both concrete and abstract subclasses can be derived from, to implement +additional test cases (see BignumCmp and BignumCmpAbs for examples of deriving +from abstract and concrete classes). -Adding test generation for a function: +Adding test case generation for a function: A subclass representing the test function should be added, deriving from a -Target class or a descendant. This subclass must set/implement the following: +file Target. This test class must set/implement the following: - test_function: the function name from the associated .function file. - - arguments(): generation of the arguments required for the test_function. - - generate_function_test(): generation of the test cases for the function. + - test_name: a descriptive name or brief summary to refer to the test + function. + - arguments(): a method to generate the list of arguments required for the + test_function. + - generate_function_test(): a method to generate TestCases for the function. + This should create instances of the class with required input data, and + call `.create_test_case()` to yield the TestCase. Additional details and other attributes/methods are given in the documentation of BaseTarget in test_generation.py. @@ -71,7 +79,7 @@ class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): class BignumOperation(BignumTarget, metaclass=ABCMeta): - """Common features for test cases covering binary bignum operations. + """Common features for bignum binary operations. This adds functionality common in binary operation tests. This includes generation of case descriptions, using descriptions of values and symbols @@ -130,7 +138,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): """Generate a description of the argument val. This produces a simple description of the value, which are used in test - case naming, to add context to the test cases. + case naming, to add context. """ if val == "": return "0 (null)" @@ -150,7 +158,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: - """Generator for pairs of inputs. + """Generator to yield pairs of inputs. Combinations are first generated from all input values, and then specific cases provided. @@ -169,7 +177,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): class BignumCmp(BignumOperation): - """Target for bignum comparison test cases.""" + """Test cases for bignum value comparison.""" count = 0 test_function = "mbedtls_mpi_cmp_mpi" test_name = "MPI compare" @@ -190,7 +198,7 @@ class BignumCmp(BignumOperation): class BignumCmpAbs(BignumCmp): - """Target for bignum comparison, absolute variant.""" + """Test cases for absolute bignum value comparison.""" count = 0 test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" @@ -200,7 +208,7 @@ class BignumCmpAbs(BignumCmp): class BignumAdd(BignumOperation): - """Target for bignum addition test cases.""" + """Test cases for bignum value addition.""" count = 0 test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" @@ -223,7 +231,7 @@ class BignumAdd(BignumOperation): class BignumTestGenerator(test_generation.TestGenerator): - """Test generator subclass including bignum targets.""" + """Test generator subclass setting bignum targets.""" TARGETS = { subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() From 9df9faac5cb0c223e30360da41f72e32497c7886 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 12:49:41 +0100 Subject: [PATCH 28/47] Use argparser default for targets Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 9e004a69f..b22d58f99 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -166,6 +166,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', + default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) build_tree.chdir_to_root() @@ -179,14 +180,11 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): print(';'.join(generator.filename_for(name) for name in sorted(generator.TARGETS)), end='') return - if options.targets: - # Allow "-" as a special case so you can run - # ``generate_xxx_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] - else: - options.targets = sorted(generator.TARGETS) + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] for target in options.targets: generator.generate_target(target) From 76f45625e660a166b29c4f5d513c2e76dac3e00b Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 13:21:45 +0100 Subject: [PATCH 29/47] Fix trailing whitespace Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index b22d58f99..2981a7470 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -166,7 +166,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', - default=sorted(generator_class.TARGETS), + default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) build_tree.chdir_to_root() From 3366ebcb66fc5e3a142cfc88d750ea3d03245518 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 16:18:22 +0100 Subject: [PATCH 30/47] Add test_generation.py dependency in builds Signed-off-by: Werner Lewis --- tests/CMakeLists.txt | 2 ++ tests/Makefile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 776d9557d..57cf9770f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -63,6 +63,7 @@ if(GEN_FILES) DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py + ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py ) add_custom_command( OUTPUT @@ -79,6 +80,7 @@ if(GEN_FILES) ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/macro_collector.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/psa_storage.py ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py + ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_config.h ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_values.h ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_extra.h diff --git a/tests/Makefile b/tests/Makefile index e9acca3fe..8777ae92f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -93,6 +93,7 @@ generated_files: $(GENERATED_FILES) $(GENERATED_BIGNUM_DATA_FILES): generated_bignum_test_data generated_bignum_test_data: scripts/generate_bignum_tests.py generated_bignum_test_data: ../scripts/mbedtls_dev/test_case.py +generated_bignum_test_data: ../scripts/mbedtls_dev/test_generation.py generated_bignum_test_data: echo " Gen $(GENERATED_BIGNUM_DATA_FILES)" $(PYTHON) scripts/generate_bignum_tests.py @@ -103,6 +104,7 @@ generated_psa_test_data: ../scripts/mbedtls_dev/crypto_knowledge.py generated_psa_test_data: ../scripts/mbedtls_dev/macro_collector.py generated_psa_test_data: ../scripts/mbedtls_dev/psa_storage.py generated_psa_test_data: ../scripts/mbedtls_dev/test_case.py +generated_psa_test_data: ../scripts/mbedtls_dev/test_generation.py ## The generated file only depends on the options that are present in ## crypto_config.h, not on which options are set. To avoid regenerating this ## file all the time when switching between configurations, don't declare From 81f24443b7888180c75554c6fa1b29c8f942ccca Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 16:27:05 +0100 Subject: [PATCH 31/47] Modify wording in docstrings Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 5 +++-- tests/scripts/generate_bignum_tests.py | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 2981a7470..e833008b5 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -35,7 +35,8 @@ T = TypeVar('T') #pylint: disable=invalid-name class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. - This should be derived from for file Targets. + Derive directly from this class when adding new file Targets, setting + `target_basename`. Attributes: count: Counter for test cases from this class. @@ -113,7 +114,7 @@ class BaseTarget(metaclass=ABCMeta): """Generate test cases for the class and its subclasses. In classes with `test_function` set, `generate_function_tests()` is - used to generate test cases first. + called to generate test cases first. In all classes, this method will iterate over its subclasses, and yield from `generate_tests()` in each. Calling this method on a class X diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 4486d4958..8a8425e1c 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -24,7 +24,8 @@ from abstract and concrete classes). Adding test case generation for a function: A subclass representing the test function should be added, deriving from a -file Target. This test class must set/implement the following: +file Target such as BignumTarget. This test class must set/implement the +following: - test_function: the function name from the associated .function file. - test_name: a descriptive name or brief summary to refer to the test function. @@ -128,8 +129,8 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): def result(self) -> str: """Get the result of the operation. - This may be calculated during initialization and stored as `_result`, - or calculated when the method is called. + This could be calculated during initialization and stored as `_result` + and then returned, or calculated when the method is called. """ raise NotImplementedError @@ -137,8 +138,8 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): def value_description(val) -> str: """Generate a description of the argument val. - This produces a simple description of the value, which are used in test - case naming, to add context. + This produces a simple description of the value, which is used in test + case naming to add context. """ if val == "": return "0 (null)" @@ -231,7 +232,7 @@ class BignumAdd(BignumOperation): class BignumTestGenerator(test_generation.TestGenerator): - """Test generator subclass setting bignum targets.""" + """Test generator subclass, for bignum file Targets.""" TARGETS = { subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() From a4b7720cb57bb9dc68d9f263bb526918dff0e4a3 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 16:55:44 +0100 Subject: [PATCH 32/47] Use `combinations_with_replacement` for inputs When generating combinations of values, `itertools.combinations` will not allow inputs to be repeated. This is replaced so that cases where input values match are generated, i.e. ("0", "0"). Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 8a8425e1c..b08ba3785 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -166,7 +166,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): """ yield from cast( Iterator[Tuple[str, str]], - itertools.combinations(cls.input_values, 2) + itertools.combinations_with_replacement(cls.input_values, 2) ) yield from cls.input_cases @@ -215,7 +215,7 @@ class BignumAdd(BignumOperation): test_name = "MPI add" input_cases = cast( List[Tuple[str, str]], - list(itertools.combinations( + list(itertools.combinations_with_replacement( [ "1c67967269c6", "9cde3", "-1c67967269c6", "-9cde3", From 466f0363264ed238be3b26dacb624c7b4ccf8eba Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 17:01:38 +0100 Subject: [PATCH 33/47] Add dependencies attribute to BaseTarget Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index e833008b5..9eaa7e28f 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -42,6 +42,7 @@ class BaseTarget(metaclass=ABCMeta): count: Counter for test cases from this class. case_description: Short description of the test case. This may be automatically generated using the class, or manually set. + dependencies: A list of dependencies required for the test case. target_basename: Basename of file to write generated tests to. This should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. @@ -51,6 +52,7 @@ class BaseTarget(metaclass=ABCMeta): """ count = 0 case_description = "" + dependencies: List[str] = [] target_basename = "" test_function = "" test_name = "" @@ -94,6 +96,7 @@ class BaseTarget(metaclass=ABCMeta): tc.set_description(self.description()) tc.set_function(self.test_function) tc.set_arguments(self.arguments()) + tc.set_dependencies(self.dependencies) return tc From aaf3b79bbbb20523dc45b1438e3e0dbde6295e33 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 17:16:44 +0100 Subject: [PATCH 34/47] Use Python 3.5 style typing for dependencies Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 9eaa7e28f..1adf8e24f 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -52,7 +52,7 @@ class BaseTarget(metaclass=ABCMeta): """ count = 0 case_description = "" - dependencies: List[str] = [] + dependencies = [] # type: List[str] target_basename = "" test_function = "" test_name = "" From a4668a6b6ca7d6a0a07ed5ff958229a619b8ac66 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 11:56:34 +0100 Subject: [PATCH 35/47] Rework TestGenerator to add file targets BaseTarget-derived targets are now added to TestGenerator.targets in initialization. This reduces repeated code in generate_xxx_tests.py scripts which use this framework. Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 29 ++++++++++++++++---------- tests/scripts/generate_bignum_tests.py | 9 +------- tests/scripts/generate_psa_tests.py | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 1adf8e24f..57eb2be1f 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -133,6 +133,11 @@ class TestGenerator: """Generate test data.""" def __init__(self, options) -> None: self.test_suite_directory = getattr(options, 'directory') + # Add file Targets which have been declared in other modules + self.targets.update({ + subclass.target_basename: subclass.generate_tests + for subclass in BaseTarget.__subclasses__() + }) def filename_for(self, basename: str) -> str: """The location of the data file with the specified base name.""" @@ -149,7 +154,7 @@ class TestGenerator: # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] + targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] def generate_target(self, name: str, *target_args) -> None: """Generate cases and write to data file for a target. @@ -157,7 +162,7 @@ class TestGenerator: For target callables which require arguments, override this function and pass these arguments using super() (see PSATestGenerator). """ - test_cases = self.TARGETS[name](*target_args) + test_cases = self.targets[name](*target_args) self.write_test_data_file(name, test_cases) def main(args, generator_class: Type[TestGenerator] = TestGenerator): @@ -170,25 +175,27 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', - default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) build_tree.chdir_to_root() generator = generator_class(options) if options.list: - for name in sorted(generator.TARGETS): + for name in sorted(generator.targets): print(generator.filename_for(name)) return # List in a cmake list format (i.e. ';'-separated) if options.list_for_cmake: print(';'.join(generator.filename_for(name) - for name in sorted(generator.TARGETS)), end='') + for name in sorted(generator.targets)), end='') return - # Allow "-" as a special case so you can run - # ``generate_xxx_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.targets) for target in options.targets: generator.generate_target(target) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index b08ba3785..f6136804b 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -231,12 +231,5 @@ class BignumAdd(BignumOperation): return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) -class BignumTestGenerator(test_generation.TestGenerator): - """Test generator subclass, for bignum file Targets.""" - TARGETS = { - subclass.target_basename: subclass.generate_tests for subclass in - test_generation.BaseTarget.__subclasses__() - } # type: Dict[str, Callable[[], Iterable[test_case.TestCase]]] - if __name__ == '__main__': - test_generation.main(sys.argv[1:], BignumTestGenerator) + test_generation.main(sys.argv[1:]) diff --git a/tests/scripts/generate_psa_tests.py b/tests/scripts/generate_psa_tests.py index 9f32655ae..81b35c9b3 100755 --- a/tests/scripts/generate_psa_tests.py +++ b/tests/scripts/generate_psa_tests.py @@ -896,7 +896,7 @@ class PSATestGenerator(test_generation.TestGenerator): """Test generator subclass including PSA targets and info.""" # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = { + targets = { 'test_suite_psa_crypto_generate_key.generated': lambda info: KeyGenerate(info).test_cases_for_key_generation(), 'test_suite_psa_crypto_not_supported.generated': From 56013081c7f26b89551c7f3ca37f74cc0bda3377 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 12:57:37 +0100 Subject: [PATCH 36/47] Remove unused imports Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 ++-- tests/scripts/generate_bignum_tests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 57eb2be1f..682f7b036 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -193,8 +193,8 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): # ``generate_xxx_tests.py - $targets`` and it works uniformly whether # ``$targets`` is empty or not. options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] + for target in options.targets + if target != '-'] else: options.targets = sorted(generator.targets) for target in options.targets: diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index f6136804b..2a8107725 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -58,7 +58,7 @@ import itertools import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterable, Iterator, List, Tuple, TypeVar, cast +from typing import Iterator, List, Tuple, TypeVar, cast import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case From 855e45c81744de3f6ff6e7991eb39f00202fa517 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 17:26:19 +0100 Subject: [PATCH 37/47] Use simpler int to hex string conversion Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 2a8107725..cc4db4c59 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -228,7 +228,7 @@ class BignumAdd(BignumOperation): self.symbol = "+" def result(self) -> str: - return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) + return quote_str("{:x}".format(self.int_l + self.int_r)) if __name__ == '__main__': From 1fade8adb603b0ebf87fbca87dab0eaba42970fa Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 12 Sep 2022 17:34:15 +0100 Subject: [PATCH 38/47] Move symbol definition out of __init__ Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index cc4db4c59..28d29bfe1 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -211,6 +211,7 @@ class BignumCmpAbs(BignumCmp): class BignumAdd(BignumOperation): """Test cases for bignum value addition.""" count = 0 + symbol = "+" test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" input_cases = cast( @@ -223,10 +224,6 @@ class BignumAdd(BignumOperation): )) ) - def __init__(self, val_l, val_r) -> None: - super().__init__(val_l, val_r) - self.symbol = "+" - def result(self) -> str: return quote_str("{:x}".format(self.int_l + self.int_r)) From 3dc45198e63c889bfca10cc718c723cef82db60e Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 12 Sep 2022 17:35:27 +0100 Subject: [PATCH 39/47] Replace L/R inputs with A/B Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 28d29bfe1..3f60a0915 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -101,14 +101,14 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): ] # type: List[str] input_cases = cast(List[Tuple[str, str]], []) # type: List[Tuple[str, str]] - def __init__(self, val_l: str, val_r: str) -> None: - self.arg_l = val_l - self.arg_r = val_r - self.int_l = hex_to_int(val_l) - self.int_r = hex_to_int(val_r) + def __init__(self, val_a: str, val_b: str) -> None: + self.arg_a = val_a + self.arg_b = val_b + self.int_a = hex_to_int(val_a) + self.int_b = hex_to_int(val_b) def arguments(self) -> List[str]: - return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] + return [quote_str(self.arg_a), quote_str(self.arg_b), self.result()] def description(self) -> str: """Generate a description for the test case. @@ -119,9 +119,9 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): """ if not self.case_description: self.case_description = "{} {} {}".format( - self.value_description(self.arg_l), + self.value_description(self.arg_a), self.symbol, - self.value_description(self.arg_r) + self.value_description(self.arg_b) ) return super().description() @@ -172,8 +172,8 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for l_value, r_value in cls.get_value_pairs(): - cur_op = cls(l_value, r_value) + for a_value, b_value in cls.get_value_pairs(): + cur_op = cls(a_value, b_value) yield cur_op.create_test_case() @@ -189,9 +189,9 @@ class BignumCmp(BignumOperation): ("2b5", "2b6") ] - def __init__(self, val_l, val_r) -> None: - super().__init__(val_l, val_r) - self._result = int(self.int_l > self.int_r) - int(self.int_l < self.int_r) + def __init__(self, val_a, val_b) -> None: + super().__init__(val_a, val_b) + self._result = int(self.int_a > self.int_b) - int(self.int_a < self.int_b) self.symbol = ["<", "==", ">"][self._result + 1] def result(self) -> str: @@ -204,8 +204,8 @@ class BignumCmpAbs(BignumCmp): test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" - def __init__(self, val_l, val_r) -> None: - super().__init__(val_l.strip("-"), val_r.strip("-")) + def __init__(self, val_a, val_b) -> None: + super().__init__(val_a.strip("-"), val_b.strip("-")) class BignumAdd(BignumOperation): @@ -225,7 +225,7 @@ class BignumAdd(BignumOperation): ) def result(self) -> str: - return quote_str("{:x}".format(self.int_l + self.int_r)) + return quote_str("{:x}".format(self.int_a + self.int_b)) if __name__ == '__main__': From 34d6d3e4e527546f1151b3cd40ee06e95d1832e0 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 12:59:32 +0100 Subject: [PATCH 40/47] Update comments/docstrings in TestGenerator Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 682f7b036..f1b268229 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -130,10 +130,12 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: - """Generate test data.""" + """Generate test cases and write to data files.""" def __init__(self, options) -> None: self.test_suite_directory = getattr(options, 'directory') - # Add file Targets which have been declared in other modules + # Update `targets` with an entry for each child class of BaseTarget. + # Each entry represents a file generated by the BaseTarget framework, + # and enables generating the .data files using the CLI. self.targets.update({ subclass.target_basename: subclass.generate_tests for subclass in BaseTarget.__subclasses__() From 858cffde1e5fe2b0b0383580ddc364011eed51ea Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 13:02:40 +0100 Subject: [PATCH 41/47] Add toggle for test case count in descriptions Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index f1b268229..81af7ba27 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -43,6 +43,7 @@ class BaseTarget(metaclass=ABCMeta): case_description: Short description of the test case. This may be automatically generated using the class, or manually set. dependencies: A list of dependencies required for the test case. + show_test_count: Toggle for inclusion of `count` in the test description. target_basename: Basename of file to write generated tests to. This should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. @@ -53,6 +54,7 @@ class BaseTarget(metaclass=ABCMeta): count = 0 case_description = "" dependencies = [] # type: List[str] + show_test_count = True target_basename = "" test_function = "" test_name = "" @@ -78,16 +80,19 @@ class BaseTarget(metaclass=ABCMeta): """Create a test case description. Creates a description of the test case, including a name for the test - function, a case number, and a description the specific test case. - This should inform a reader what is being tested, and provide context - for the test case. + function, an optional case count, and a description of the specific + test case. This should inform a reader what is being tested, and + provide context for the test case. Returns: Description for the test case. """ - return "{} #{} {}".format( - self.test_name, self.count, self.case_description - ).strip() + if self.show_test_count: + return "{} #{} {}".format( + self.test_name, self.count, self.case_description + ).strip() + else: + return "{} {}".format(self.test_name, self.case_description).strip() def create_test_case(self) -> test_case.TestCase: From 00d02423a58eba33ddaa8d56c1c3ce06e607ba53 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 13:39:20 +0100 Subject: [PATCH 42/47] Remove argparser default for directory This reverts commit f156c43702bec949c80a56a993a0d5009e39ab17. Adds a comment to explain reasoning for current implementation. Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 81af7ba27..87cb43ecd 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -137,7 +137,7 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: """Generate test cases and write to data files.""" def __init__(self, options) -> None: - self.test_suite_directory = getattr(options, 'directory') + self.test_suite_directory = getattr(options, 'directory', 'tests/suites') # Update `targets` with an entry for each child class of BaseTarget. # Each entry represents a file generated by the BaseTarget framework, # and enables generating the .data files using the CLI. @@ -179,8 +179,12 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): help='List available targets and exit') parser.add_argument('--list-for-cmake', action='store_true', help='Print \';\'-separated list of available targets and exit') - parser.add_argument('--directory', default="tests/suites", metavar='DIR', + parser.add_argument('--directory', metavar='DIR', help='Output directory (default: tests/suites)') + # The `--directory` option is interpreted relative to the directory from + # which the script is invoked, but the default is relative to the root of + # the mbedtls tree. The default should not be set above, but instead after + # `build_tree.chdir_to_root()` is called. parser.add_argument('targets', nargs='*', metavar='TARGET', help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) From b6e809133d5f6b07a3f8b77bea8aef21988308a1 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 15:00:22 +0100 Subject: [PATCH 43/47] Use typing.cast instead of unqualified cast Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 3f60a0915..3453b6bc3 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -56,9 +56,10 @@ of BaseTarget in test_generation.py. import itertools import sys +import typing from abc import ABCMeta, abstractmethod -from typing import Iterator, List, Tuple, TypeVar, cast +from typing import Iterator, List, Tuple, TypeVar import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -99,7 +100,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" ] # type: List[str] - input_cases = cast(List[Tuple[str, str]], []) # type: List[Tuple[str, str]] + input_cases = [] # type: List[Tuple[str, str]] def __init__(self, val_a: str, val_b: str) -> None: self.arg_a = val_a @@ -164,7 +165,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): Combinations are first generated from all input values, and then specific cases provided. """ - yield from cast( + yield from typing.cast( Iterator[Tuple[str, str]], itertools.combinations_with_replacement(cls.input_values, 2) ) @@ -214,7 +215,7 @@ class BignumAdd(BignumOperation): symbol = "+" test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" - input_cases = cast( + input_cases = typing.cast( List[Tuple[str, str]], list(itertools.combinations_with_replacement( [ From ac446c8a04b9dacdd546b791e87e4de5eed2427a Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 15:12:46 +0100 Subject: [PATCH 44/47] Add combination_pairs helper function Wrapper function for itertools.combinations_with_replacement, with explicit cast due to imprecise typing with older versions of mypy. Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 3453b6bc3..d156f56f8 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -73,6 +73,17 @@ def hex_to_int(val: str) -> int: def quote_str(val) -> str: return "\"{}\"".format(val) +def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: + """Return all pair combinations from input values. + + The return value is cast, as older versions of mypy are unable to derive + the specific type returned by itertools.combinations_with_replacement. + """ + return typing.cast( + List[Tuple[T, T]], + list(itertools.combinations_with_replacement(values, 2)) + ) + class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method @@ -165,10 +176,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): Combinations are first generated from all input values, and then specific cases provided. """ - yield from typing.cast( - Iterator[Tuple[str, str]], - itertools.combinations_with_replacement(cls.input_values, 2) - ) + yield from combination_pairs(cls.input_values) yield from cls.input_cases @classmethod @@ -215,14 +223,11 @@ class BignumAdd(BignumOperation): symbol = "+" test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" - input_cases = typing.cast( - List[Tuple[str, str]], - list(itertools.combinations_with_replacement( - [ - "1c67967269c6", "9cde3", - "-1c67967269c6", "-9cde3", - ], 2 - )) + input_cases = combination_pairs( + [ + "1c67967269c6", "9cde3", + "-1c67967269c6", "-9cde3", + ] ) def result(self) -> str: From 52ae326ebb557612592cd8a0fcd08258f16b059f Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 16:26:54 +0100 Subject: [PATCH 45/47] Update references to file targets in docstrings Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 5 +++-- tests/scripts/generate_bignum_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 87cb43ecd..c9a73c4ad 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -35,8 +35,9 @@ T = TypeVar('T') #pylint: disable=invalid-name class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. - Derive directly from this class when adding new file Targets, setting - `target_basename`. + Child classes of this class represent an output file, and can be referred + to as file targets. These indicate where test cases will be written to for + all subclasses of the file target, which is set by `target_basename`. Attributes: count: Counter for test cases from this class. diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index d156f56f8..b4915d846 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -6,11 +6,11 @@ generate only the specified files. Class structure: -Child classes of test_generation.BaseTarget (file Targets) represent a target +Child classes of test_generation.BaseTarget (file targets) represent an output file. These indicate where test cases will be written to, for all subclasses of -this Target. Multiple Target classes should not reuse a `target_basename`. +this target. Multiple file targets should not reuse a `target_basename`. -Each subclass derived from a file Target can either be: +Each subclass derived from a file target can either be: - A concrete class, representing a test function, which generates test cases. - An abstract class containing shared methods and attributes, not associated with a test function. An example is BignumOperation, which provides @@ -24,7 +24,7 @@ from abstract and concrete classes). Adding test case generation for a function: A subclass representing the test function should be added, deriving from a -file Target such as BignumTarget. This test class must set/implement the +file target such as BignumTarget. This test class must set/implement the following: - test_function: the function name from the associated .function file. - test_name: a descriptive name or brief summary to refer to the test From 07c830c1647278ec5f446647b9122097bd5a212f Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 15 Sep 2022 09:02:07 +0100 Subject: [PATCH 46/47] Fix setting for default test suite directory Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index c9a73c4ad..a82f79e67 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -138,7 +138,8 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: """Generate test cases and write to data files.""" def __init__(self, options) -> None: - self.test_suite_directory = getattr(options, 'directory', 'tests/suites') + self.test_suite_directory = self.get_option(options, 'directory', + 'tests/suites') # Update `targets` with an entry for each child class of BaseTarget. # Each entry represents a file generated by the BaseTarget framework, # and enables generating the .data files using the CLI. @@ -147,6 +148,11 @@ class TestGenerator: for subclass in BaseTarget.__subclasses__() }) + @staticmethod + def get_option(options, name: str, default: T) -> T: + value = getattr(options, name, None) + return default if value is None else value + def filename_for(self, basename: str) -> str: """The location of the data file with the specified base name.""" return posixpath.join(self.test_suite_directory, basename + '.data') From c2fb540c67cf58642829c4ddffe74ea9badb7f35 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 16 Sep 2022 17:03:54 +0100 Subject: [PATCH 47/47] Use a script specific description in CLI help Previous changes used the docstring of the test_generation module, which does not inform a user about the script. Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 ++-- tests/scripts/generate_bignum_tests.py | 4 ++-- tests/scripts/generate_psa_tests.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index a82f79e67..a88425f46 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -179,9 +179,9 @@ class TestGenerator: test_cases = self.targets[name](*target_args) self.write_test_data_file(name, test_cases) -def main(args, generator_class: Type[TestGenerator] = TestGenerator): +def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator): """Command line entry point.""" - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser(description=description) parser.add_argument('--list', action='store_true', help='List available targets and exit') parser.add_argument('--list-for-cmake', action='store_true', diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index b4915d846..ceafa4a48 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -233,6 +233,6 @@ class BignumAdd(BignumOperation): def result(self) -> str: return quote_str("{:x}".format(self.int_a + self.int_b)) - if __name__ == '__main__': - test_generation.main(sys.argv[1:]) + # Use the section of the docstring relevant to the CLI as description + test_generation.main(sys.argv[1:], "\n".join(__doc__.splitlines()[:4])) diff --git a/tests/scripts/generate_psa_tests.py b/tests/scripts/generate_psa_tests.py index 81b35c9b3..c788fd76b 100755 --- a/tests/scripts/generate_psa_tests.py +++ b/tests/scripts/generate_psa_tests.py @@ -917,4 +917,4 @@ class PSATestGenerator(test_generation.TestGenerator): super().generate_target(name, self.info) if __name__ == '__main__': - test_generation.main(sys.argv[1:], PSATestGenerator) + test_generation.main(sys.argv[1:], __doc__, PSATestGenerator)