From b4063890e87d401b4d46cc19a7e84fc44288d4c8 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 21:36:44 +0200 Subject: [PATCH 01/28] Mbed TLS configuration file manipulation library and tool This is meant to be a drop-in replacement for config.pl which can additionally be used as a library in a Python script. So far this script supports the commands 'get', 'set' and 'realfull' but not the other built-in configurations. --- scripts/config.py | 298 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100755 scripts/config.py diff --git a/scripts/config.py b/scripts/config.py new file mode 100755 index 000000000..f86d64b69 --- /dev/null +++ b/scripts/config.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 + +"""Mbed TLS configuration file manipulation library and tool + +Basic usage, to read the Mbed TLS or Mbed Crypto configuration: + config = ConfigFile() + if 'MBEDTLS_RSA_C' in config: print('RSA is enabled') +""" + +## Copyright (C) 2019, ARM Limited, All Rights Reserved +## SPDX-License-Identifier: Apache-2.0 +## +## Licensed under the Apache License, Version 2.0 (the "License"); you may +## not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +## This file is part of Mbed TLS (https://tls.mbed.org) + +import re + +class Setting: + """Representation of one Mbed TLS config.h setting. + + Fields: + * name: the symbol name ('MBEDTLS_xxx'). + * value: the value of the macro. The empty string for a plain #define + with no value. + * active: True if name is defined, False if a #define for name is + present in config.h but commented out. + """ + # pylint: disable=too-few-public-methods + def __init__(self, active, name, value=''): + self.active = active + self.name = name + self.value = value + +class Config: + """Representation of the Mbed TLS configuration. + + In the documentation of this class, a symbol is said to be *active* + if there is a #define for it that is not commented out, and *known* + if there is a #define for it whether commented out or not. + + This class supports the following protocols: + * `name in config` is True if the symbol `name` is set in the + configuration, False otherwise (whether `name` is known but commented + out or not known at all). + * `config[name]` is the value of the macro `name`. If `name` is not + set, raise `KeyError` (even if a definition for `name` is present + but commented out). + * `config[name] = value` sets the value associated to `name`. `name` + must be known, but does not need to be set. This does not cause + name to become set. + """ + + def __init__(self): + self.settings = {} + + def __contains__(self, name): + """True if the given symbol is active (i.e. set). + + False if the given symbol is not set, even if a definition + is present but commented out. + """ + return name in self.settings and self.settings[name].active + + def all(self, *names): + """True if all the elements of names are active (i.e. set).""" + return all(self.__contains__(name) for name in names) + + def any(self, *names): + """True if at least one symbol in names are active (i.e. set).""" + return any(self.__contains__(name) for name in names) + + def known(self, name): + """True if a #define for name is present, whether it's commented out or not.""" + return name in self.settings + + def __getitem__(self, name): + """Get the value of name, i.e. what the preprocessor symbol expands to. + + If name is not known, raise KeyError. name does not need to be active. + """ + return self.settings[name].value + + def get(self, name, default=None): + """Get the value of name. If name is inactive (not set), return default. + + If a #define for name is present and not commented out, return + its expansion, even if this is the empty string. + + If a #define for name is present but commented out, return default. + """ + if name in self.settings: + return self.settings[name].value + else: + return default + + def __setitem__(self, name, value): + """If name is known, set its value. + + If name is not known, raise KeyError. + """ + self.settings[name].value = value + + def set(self, name, value=None): + """Set name to the given value and make it active. + + If value is None and name is already known, don't change its value. + If value is None and name is not known, set its value to the empty + string. + """ + if name in self.settings: + if value is not None: + self.settings[name].value = value + self.settings[name].active = True + else: + self.settings[name] = Setting(True, name, value=value) + + def unset(self, name): + """Make name unset (inactive). + + name remains known. + """ + self.set(name) + self.settings[name].active = False + + def adapt(self, adapter): + """Run adapter on each known symbol and (de)activate it accordingly. + + `adapter` must be a function that returns a boolean. It is called as + `adapter(name, active)` for each setting, where `active` is `True` + if `name` is set and `False` if `name` is known but unset. If + `adapter` returns `True`, then set `name` (i.e. make it active), + otherwise unset `name` (i.e. make it known but inactive). + """ + for setting in self.settings.values(): + setting.active = adapter(setting.name, setting.active) + +def realfull_adapter(_name, _set): + """Uncomment everything.""" + return True + +class ConfigFile(Config): + """Representation of the Mbed TLS configuration read for a file. + + See the documentation of the `Config` class for methods to query + and modify the configuration. + """ + + default_path = 'include/mbedtls/config.h' + + def __init__(self, filename=None): + """Read the Mbed TLS configuration file.""" + if filename is None: + filename = self.default_path + super().__init__() + self.filename = filename + with open(filename) as file: + self.templates = [self._parse_line(line) for line in file] + + def set(self, name, value=None): + if name not in self.settings: + self.templates.append((name, '', '#define ' + name + ' ')) + super().set(name, value) + + _define_line_regexp = (r'(?P\s*)' + + r'(?P(//\s*)?)' + + r'(?P#\s*define\s+)' + + r'(?P\w+)' + + r'(?P(?:\((?:\w|\s|,)*\))?)' + + r'(?P\s*)' + + r'(?P.*)') + def _parse_line(self, line): + """Parse a line in config.h and return the corresponding template.""" + line = line.rstrip('\r\n') + m = re.match(self._define_line_regexp, line) + if m: + active = not m.group('commented_out') + name = m.group('name') + value = m.group('value') + template = (name, + m.group('indentation'), + m.group('define') + name + + m.group('arguments') + m.group('separator')) + self.settings[name] = Setting(active, name, value) + return template + else: + return line + + def _format_template(self, name, indent, middle): + """Build a line for config.h for the given setting. + + The line has the form "#define ". + """ + setting = self.settings[name] + return ''.join([indent, + '' if setting.active else '//', + middle, + setting.value]).rstrip() + + def write_to_stream(self, output): + """Write the whole configuration to output.""" + for template in self.templates: + if isinstance(template, str): + line = template + else: + line = self._format_template(*template) + output.write(line + '\n') + + def write(self, filename=None): + """Write the whole configuration to the file it was read from. + + If filename is specified, write to this file instead. + """ + if filename is None: + filename = self.filename + with open(filename, 'w') as output: + self.write_to_stream(output) + +if __name__ == '__main__': + def main(): + """Command line config.h manipulation tool.""" + parser = argparse.ArgumentParser(description=""" + Mbed TLS and Mbed Crypto configuration file manipulation tool. + """) + parser.add_argument('--file', '-f', + help="""File to read (and modify if requested). + Default: {}. + """.format(ConfigFile.default_path)) + parser.add_argument('--force', '-o', + help="""For the set command, if SYMBOL is not + present, add a definition for it.""") + subparsers = parser.add_subparsers(dest='command', + title='Commands') + parser_get = subparsers.add_parser('get', + help="""Find the value of SYMBOL + and print it. Exit with + status 0 if a #define for SYMBOL is + found, 1 otherwise. + """) + parser_get.add_argument('symbol', metavar='SYMBOL') + parser_set = subparsers.add_parser('set', + help="""Set SYMBOL to VALUE. + If VALUE is omitted, just uncomment + the #define for SYMBOL. + Error out of a line defining + SYMBOL (commented or not) is not + found, unless --force is passed. + """) + parser_set.add_argument('symbol', metavar='SYMBOL') + parser_set.add_argument('value', metavar='VALUE', nargs='?') + parser_unset = subparsers.add_parser('unset', + help="""Comment out the #define + for SYMBOL. Do nothing if none + is present.""") + parser_unset.add_argument('symbol', metavar='SYMBOL') + + def add_adapter(name, function, description): + subparser = subparsers.add_parser(name, help=description) + subparser.set_defaults(adapter=function) + add_adapter('realfull', realfull_adapter, + """Uncomment all #defines. No exceptions.""") + + args = parser.parse_args() + config = ConfigFile(args.file) + if args.command == 'get': + if args.symbol in config: + value = config[args.symbol] + if value: + sys.stdout.write(value + '\n') + return args.symbol not in config + elif args.command == 'set': + if not args.force and args.symbol not in config: + sys.stderr.write("A #define for the symbol {} " + "was not found in {}" + .format(args.symbol, args.file)) + return 1 + config.set(args.symbol, value=args.value) + elif args.command == 'unset': + config.unset(args.symbol) + else: + config.adapt(args.adapter) + config.write() + + # Import modules only used by main only if main is defined and called. + # pylint: disable=wrong-import-position + import argparse + import sys + sys.exit(main()) From 53d41ae8722578c8f3b39e08929c6354211c9c61 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 23:31:53 +0200 Subject: [PATCH 02/28] Implement the 'full' and 'baremetal' configurations Also fix 'realfull' to only affect the appropriate sections. Tested to produce the same results as config.pl on the default configuration. This commit deliberately contains a direct copy the lists of symbol names from config.pl. --- scripts/config.py | 127 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 115 insertions(+), 12 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index f86d64b69..4f5edf8d3 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -35,12 +35,14 @@ class Setting: with no value. * active: True if name is defined, False if a #define for name is present in config.h but commented out. + * section: the name of the section that contains this symbol. """ # pylint: disable=too-few-public-methods - def __init__(self, active, name, value=''): + def __init__(self, active, name, value='', section=None): self.active = active self.name = name self.value = value + self.section = section class Config: """Representation of the Mbed TLS configuration. @@ -137,18 +139,100 @@ class Config: """Run adapter on each known symbol and (de)activate it accordingly. `adapter` must be a function that returns a boolean. It is called as - `adapter(name, active)` for each setting, where `active` is `True` - if `name` is set and `False` if `name` is known but unset. If + `adapter(name, active, section)` for each setting, where `active` is + `True` if `name` is set and `False` if `name` is known but unset, + and `section` is the name of the section containing `name`. If `adapter` returns `True`, then set `name` (i.e. make it active), otherwise unset `name` (i.e. make it known but inactive). """ for setting in self.settings.values(): - setting.active = adapter(setting.name, setting.active) + setting.active = adapter(setting.name, setting.active, + setting.section) -def realfull_adapter(_name, _set): - """Uncomment everything.""" +def is_full_section(section): + """Is this section affected by "config.py full" and friends?""" + return section.endswith('support') or section.endswith('modules') + +def realfull_adapter(_name, active, section): + """Uncomment everything in the system and feature sections.""" + if not is_full_section(section): + return active return True +def include_in_full(name): + """Rules for symbols in the "full" configuration.""" + if re.search(r'PLATFORM_[A-Z0-9]+_ALT', name): + return True + if name in [ + 'MBEDTLS_TEST_NULL_ENTROPY', + 'MBEDTLS_DEPRECATED_REMOVED', + 'MBEDTLS_HAVE_SSE2', + 'MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', + 'MBEDTLS_ECP_DP_M221_ENABLED', + 'MBEDTLS_ECP_DP_M383_ENABLED', + 'MBEDTLS_ECP_DP_M511_ENABLED', + 'MBEDTLS_MEMORY_DEBUG', + 'MBEDTLS_MEMORY_BACKTRACE', + 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', + 'MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES', + 'MBEDTLS_NO_PLATFORM_ENTROPY', + 'MBEDTLS_RSA_NO_CRT', + 'MBEDTLS_REMOVE_ARC4_CIPHERSUITES', + 'MBEDTLS_REMOVE_3DES_CIPHERSUITES', + 'MBEDTLS_SSL_HW_RECORD_ACCEL', + 'MBEDTLS_X509_ALLOW_EXTENSIONS_NON_V3', + 'MBEDTLS_X509_ALLOW_UNSUPPORTED_CRITICAL_EXTENSION', + 'MBEDTLS_ZLIB_SUPPORT', + 'MBEDTLS_PKCS11_C', + 'MBEDTLS_NO_UDBL_DIVISION', + 'MBEDTLS_NO_64BIT_MULTIPLICATION', + 'MBEDTLS_PSA_CRYPTO_SPM', + 'MBEDTLS_PSA_INJECT_ENTROPY', + 'MBEDTLS_ECP_RESTARTABLE', + 'MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED', + ]: + return False + if name.endswith('_ALT'): + return False + return True + +def full_adapter(name, active, section): + """Config adapter for "full".""" + if not is_full_section(section): + return active + return include_in_full(name) + +def keep_in_baremetal(name): + """Rules for symbols in the "baremetal" configuration.""" + if name in [ + 'MBEDTLS_NET_C', + 'MBEDTLS_TIMING_C', + 'MBEDTLS_FS_IO', + 'MBEDTLS_ENTROPY_NV_SEED', + 'MBEDTLS_HAVE_TIME', + 'MBEDTLS_HAVE_TIME_DATE', + 'MBEDTLS_DEPRECATED_WARNING', + 'MBEDTLS_HAVEGE_C', + 'MBEDTLS_THREADING_C', + 'MBEDTLS_THREADING_PTHREAD', + 'MBEDTLS_MEMORY_BACKTRACE', + 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', + 'MBEDTLS_PLATFORM_TIME_ALT', + 'MBEDTLS_PLATFORM_FPRINTF_ALT', + 'MBEDTLS_PSA_ITS_FILE_C', + 'MBEDTLS_PSA_CRYPTO_STORAGE_C', + ]: + return False + return True + +def baremetal_adapter(name, active, section): + """Config adapter for "baremetal".""" + if not is_full_section(section): + return active + if name == 'MBEDTLS_NO_PLATFORM_ENTROPY': + return True + return include_in_full(name) and keep_in_baremetal(name) + class ConfigFile(Config): """Representation of the Mbed TLS configuration read for a file. @@ -164,8 +248,10 @@ class ConfigFile(Config): filename = self.default_path super().__init__() self.filename = filename + self.current_section = 'header' with open(filename) as file: self.templates = [self._parse_line(line) for line in file] + self.current_section = None def set(self, name, value=None): if name not in self.settings: @@ -179,11 +265,20 @@ class ConfigFile(Config): r'(?P(?:\((?:\w|\s|,)*\))?)' + r'(?P\s*)' + r'(?P.*)') + _section_line_regexp = (r'\s*/?\*+\s*[\\@]name\s+SECTION:\s*' + + r'(?P
.*)[ */]*') + _config_line_regexp = re.compile(r'|'.join([_define_line_regexp, + _section_line_regexp])) def _parse_line(self, line): """Parse a line in config.h and return the corresponding template.""" line = line.rstrip('\r\n') - m = re.match(self._define_line_regexp, line) - if m: + m = re.match(self._config_line_regexp, line) + if m is None: + return line + elif m.group('section'): + self.current_section = m.group('section') + return line + else: active = not m.group('commented_out') name = m.group('name') value = m.group('value') @@ -191,10 +286,9 @@ class ConfigFile(Config): m.group('indentation'), m.group('define') + name + m.group('arguments') + m.group('separator')) - self.settings[name] = Setting(active, name, value) + self.settings[name] = Setting(active, name, value, + self.current_section) return template - else: - return line def _format_template(self, name, indent, middle): """Build a line for config.h for the given setting. @@ -267,8 +361,17 @@ if __name__ == '__main__': def add_adapter(name, function, description): subparser = subparsers.add_parser(name, help=description) subparser.set_defaults(adapter=function) + add_adapter('baremetal', baremetal_adapter, + """Like full, but exclude features that require platform + features such as file input-output.""") + add_adapter('full', full_adapter, + """Uncomment most features. + Exclude alternative implementations and platform support + options, as well as some options that are awkward to test. + """) add_adapter('realfull', realfull_adapter, - """Uncomment all #defines. No exceptions.""") + """Uncomment all boolean #defines. + Suitable for generating documentation, but not for building.""") args = parser.parse_args() config = ConfigFile(args.file) From 6c2d078935b04fefc4576c8644321304a36dc08f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 23:37:06 +0200 Subject: [PATCH 03/28] Remove obsolete options from config.py These options haven't existed for a long time. --- scripts/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index 4f5edf8d3..c19222a88 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -168,9 +168,6 @@ def include_in_full(name): 'MBEDTLS_DEPRECATED_REMOVED', 'MBEDTLS_HAVE_SSE2', 'MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', - 'MBEDTLS_ECP_DP_M221_ENABLED', - 'MBEDTLS_ECP_DP_M383_ENABLED', - 'MBEDTLS_ECP_DP_M511_ENABLED', 'MBEDTLS_MEMORY_DEBUG', 'MBEDTLS_MEMORY_BACKTRACE', 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', From 2d89ccced546f7dbf08fa9d0eb92b36b123e9a61 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 23:37:47 +0200 Subject: [PATCH 04/28] Sort symbol lists in alphabetical order They're easier to maintain that way. The old lists were partly alphabetized, partly based on config.h order, and partly in the order in which symbols had been added to config.pl. --- scripts/config.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index c19222a88..a1d932f4f 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -164,29 +164,29 @@ def include_in_full(name): if re.search(r'PLATFORM_[A-Z0-9]+_ALT', name): return True if name in [ - 'MBEDTLS_TEST_NULL_ENTROPY', 'MBEDTLS_DEPRECATED_REMOVED', + 'MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED', + 'MBEDTLS_ECP_RESTARTABLE', 'MBEDTLS_HAVE_SSE2', - 'MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', - 'MBEDTLS_MEMORY_DEBUG', 'MBEDTLS_MEMORY_BACKTRACE', 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', + 'MBEDTLS_MEMORY_DEBUG', + 'MBEDTLS_NO_64BIT_MULTIPLICATION', 'MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES', 'MBEDTLS_NO_PLATFORM_ENTROPY', - 'MBEDTLS_RSA_NO_CRT', - 'MBEDTLS_REMOVE_ARC4_CIPHERSUITES', + 'MBEDTLS_NO_UDBL_DIVISION', + 'MBEDTLS_PKCS11_C', + 'MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', + 'MBEDTLS_PSA_CRYPTO_SPM', + 'MBEDTLS_PSA_INJECT_ENTROPY', 'MBEDTLS_REMOVE_3DES_CIPHERSUITES', + 'MBEDTLS_REMOVE_ARC4_CIPHERSUITES', + 'MBEDTLS_RSA_NO_CRT', 'MBEDTLS_SSL_HW_RECORD_ACCEL', + 'MBEDTLS_TEST_NULL_ENTROPY', 'MBEDTLS_X509_ALLOW_EXTENSIONS_NON_V3', 'MBEDTLS_X509_ALLOW_UNSUPPORTED_CRITICAL_EXTENSION', 'MBEDTLS_ZLIB_SUPPORT', - 'MBEDTLS_PKCS11_C', - 'MBEDTLS_NO_UDBL_DIVISION', - 'MBEDTLS_NO_64BIT_MULTIPLICATION', - 'MBEDTLS_PSA_CRYPTO_SPM', - 'MBEDTLS_PSA_INJECT_ENTROPY', - 'MBEDTLS_ECP_RESTARTABLE', - 'MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED', ]: return False if name.endswith('_ALT'): @@ -202,22 +202,22 @@ def full_adapter(name, active, section): def keep_in_baremetal(name): """Rules for symbols in the "baremetal" configuration.""" if name in [ - 'MBEDTLS_NET_C', - 'MBEDTLS_TIMING_C', - 'MBEDTLS_FS_IO', + 'MBEDTLS_DEPRECATED_WARNING', 'MBEDTLS_ENTROPY_NV_SEED', + 'MBEDTLS_FS_IO', + 'MBEDTLS_HAVEGE_C', 'MBEDTLS_HAVE_TIME', 'MBEDTLS_HAVE_TIME_DATE', - 'MBEDTLS_DEPRECATED_WARNING', - 'MBEDTLS_HAVEGE_C', - 'MBEDTLS_THREADING_C', - 'MBEDTLS_THREADING_PTHREAD', 'MBEDTLS_MEMORY_BACKTRACE', 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', - 'MBEDTLS_PLATFORM_TIME_ALT', + 'MBEDTLS_NET_C', 'MBEDTLS_PLATFORM_FPRINTF_ALT', - 'MBEDTLS_PSA_ITS_FILE_C', + 'MBEDTLS_PLATFORM_TIME_ALT', 'MBEDTLS_PSA_CRYPTO_STORAGE_C', + 'MBEDTLS_PSA_ITS_FILE_C', + 'MBEDTLS_THREADING_C', + 'MBEDTLS_THREADING_PTHREAD', + 'MBEDTLS_TIMING_C', ]: return False return True From 5639aef7d70430d50a361b830a19addccdbc6337 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 23:39:56 +0200 Subject: [PATCH 05/28] Uniformize whitespace in commented-out defines --- include/mbedtls/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mbedtls/config.h b/include/mbedtls/config.h index 634873522..058969b73 100644 --- a/include/mbedtls/config.h +++ b/include/mbedtls/config.h @@ -3518,7 +3518,7 @@ * on it, and considering stronger message digests instead. * */ -// #define MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES +//#define MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES /** * Allow SHA-1 in the default TLS configuration for TLS 1.2 handshake From 40f103cea297f13adf58f958f74158cb7c511184 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 23:44:01 +0200 Subject: [PATCH 06/28] Support writing to a different file --- scripts/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/config.py b/scripts/config.py index a1d932f4f..9fe2e516a 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -330,6 +330,8 @@ if __name__ == '__main__': parser.add_argument('--force', '-o', help="""For the set command, if SYMBOL is not present, add a definition for it.""") + parser.add_argument('--write', '-w', + help="""File to write to instead of the input file.""") subparsers = parser.add_subparsers(dest='command', title='Commands') parser_get = subparsers.add_parser('get', @@ -389,7 +391,7 @@ if __name__ == '__main__': config.unset(args.symbol) else: config.adapt(args.adapter) - config.write() + config.write(args.write) # Import modules only used by main only if main is defined and called. # pylint: disable=wrong-import-position From 5d46f6a89b25603f0a77466c618213200c328510 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 23:52:53 +0200 Subject: [PATCH 07/28] Invoke config.py instead of config.pl git grep -Fl /config.pl | xargs sed -i -e 's!/config\.pl!/config.py!g' Also: * Change one comment in include/mbedtls/check_config.h. * Change PERL to PYTHON in CMakeLists.txt. --- CMakeLists.txt | 6 +- Makefile | 4 +- README.md | 2 +- include/mbedtls/check_config.h | 2 +- programs/fuzz/README.md | 2 +- scripts/apidoc_full.sh | 2 +- scripts/ecc-heap.sh | 4 +- scripts/footprint.sh | 8 +- scripts/memory.sh | 4 +- tests/scripts/all.sh | 198 +++++++++++++++--------------- tests/scripts/basic-build-test.sh | 4 +- tests/scripts/curves.pl | 4 +- tests/scripts/depends-hashes.pl | 4 +- tests/scripts/depends-pkalgs.pl | 4 +- tests/scripts/key-exchanges.pl | 4 +- tests/scripts/list-symbols.sh | 2 +- tests/ssl-opt.sh | 8 +- 17 files changed, 131 insertions(+), 131 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb1de8174..7f2b502e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,17 +53,17 @@ set(CTR_DRBG_128_BIT_KEY_WARNING "${WARNING_BORDER}" find_package(PythonInterp) find_package(Perl) -if(PERL_FOUND) +if(PYTHON_FOUND) # If 128-bit keys are configured for CTR_DRBG, display an appropriate warning - execute_process(COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/config.pl -f ${CMAKE_CURRENT_SOURCE_DIR}/include/mbedtls/config.h get MBEDTLS_CTR_DRBG_USE_128_BIT_KEY + execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/config.py -f ${CMAKE_CURRENT_SOURCE_DIR}/include/mbedtls/config.h get MBEDTLS_CTR_DRBG_USE_128_BIT_KEY RESULT_VARIABLE result) if(${result} EQUAL 0) message(WARNING ${CTR_DRBG_128_BIT_KEY_WARNING}) endif() # If NULL Entropy is configured, display an appropriate warning - execute_process(COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/config.pl -f ${CMAKE_CURRENT_SOURCE_DIR}/include/mbedtls/config.h get MBEDTLS_TEST_NULL_ENTROPY + execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/config.py -f ${CMAKE_CURRENT_SOURCE_DIR}/include/mbedtls/config.h get MBEDTLS_TEST_NULL_ENTROPY RESULT_VARIABLE result) if(${result} EQUAL 0) message(WARNING ${NULL_ENTROPY_WARNING}) diff --git a/Makefile b/Makefile index e898975db..4f4e43793 100644 --- a/Makefile +++ b/Makefile @@ -80,11 +80,11 @@ post_build: ifndef WINDOWS # If 128-bit keys are configured for CTR_DRBG, display an appropriate warning - -scripts/config.pl get MBEDTLS_CTR_DRBG_USE_128_BIT_KEY && ([ $$? -eq 0 ]) && \ + -scripts/config.py get MBEDTLS_CTR_DRBG_USE_128_BIT_KEY && ([ $$? -eq 0 ]) && \ echo '$(CTR_DRBG_128_BIT_KEY_WARNING)' # If NULL Entropy is configured, display an appropriate warning - -scripts/config.pl get MBEDTLS_TEST_NULL_ENTROPY && ([ $$? -eq 0 ]) && \ + -scripts/config.py get MBEDTLS_TEST_NULL_ENTROPY && ([ $$? -eq 0 ]) && \ echo '$(NULL_ENTROPY_WARNING)' endif diff --git a/README.md b/README.md index 83b32347e..43065b17d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ README for Mbed TLS Configuration ------------- -Mbed TLS should build out of the box on most systems. Some platform specific options are available in the fully documented configuration file `include/mbedtls/config.h`, which is also the place where features can be selected. This file can be edited manually, or in a more programmatic way using the Perl script `scripts/config.pl` (use `--help` for usage instructions). +Mbed TLS should build out of the box on most systems. Some platform specific options are available in the fully documented configuration file `include/mbedtls/config.h`, which is also the place where features can be selected. This file can be edited manually, or in a more programmatic way using the Perl script `scripts/config.py` (use `--help` for usage instructions). Compiler options can be set using conventional environment variables such as `CC` and `CFLAGS` when using the Make and CMake build system (see below). diff --git a/include/mbedtls/check_config.h b/include/mbedtls/check_config.h index 1bc470bc9..6ebe6b7b6 100644 --- a/include/mbedtls/check_config.h +++ b/include/mbedtls/check_config.h @@ -45,7 +45,7 @@ #endif /* Fix the config here. Not convenient to put an #ifdef _WIN32 in config.h as - * it would confuse config.pl. */ + * it would confuse config.py. */ #if !defined(MBEDTLS_PLATFORM_SNPRINTF_ALT) && \ !defined(MBEDTLS_PLATFORM_SNPRINTF_MACRO) #define MBEDTLS_PLATFORM_SNPRINTF_ALT diff --git a/programs/fuzz/README.md b/programs/fuzz/README.md index 8b24908f9..b6a433394 100644 --- a/programs/fuzz/README.md +++ b/programs/fuzz/README.md @@ -26,7 +26,7 @@ And you can run any of the fuzz targets like `fuzz_client`. To run the fuzz targets without oss-fuzz, you first need to install one libFuzzingEngine (libFuzzer for instance). Then you need to compile the code with the compiler flags of the wished sanitizer. ``` -perl scripts/config.pl set MBEDTLS_PLATFORM_TIME_ALT +perl scripts/config.py set MBEDTLS_PLATFORM_TIME_ALT mkdir build cd build cmake .. diff --git a/scripts/apidoc_full.sh b/scripts/apidoc_full.sh index bebab103e..dfe117710 100755 --- a/scripts/apidoc_full.sh +++ b/scripts/apidoc_full.sh @@ -19,7 +19,7 @@ fi CONFIG_BAK=${CONFIG_H}.bak cp -p $CONFIG_H $CONFIG_BAK -scripts/config.pl realfull +scripts/config.py realfull make apidoc mv $CONFIG_BAK $CONFIG_H diff --git a/scripts/ecc-heap.sh b/scripts/ecc-heap.sh index 94a04cf7e..69777a62c 100755 --- a/scripts/ecc-heap.sh +++ b/scripts/ecc-heap.sh @@ -59,8 +59,8 @@ EOF for F in 0 1; do for W in 2 3 4 5 6; do - scripts/config.pl set MBEDTLS_ECP_WINDOW_SIZE $W - scripts/config.pl set MBEDTLS_ECP_FIXED_POINT_OPTIM $F + scripts/config.py set MBEDTLS_ECP_WINDOW_SIZE $W + scripts/config.py set MBEDTLS_ECP_FIXED_POINT_OPTIM $F make benchmark >/dev/null 2>&1 echo "fixed point optim = $F, max window size = $W" echo "--------------------------------------------" diff --git a/scripts/footprint.sh b/scripts/footprint.sh index c08ef1c90..961a0d60b 100755 --- a/scripts/footprint.sh +++ b/scripts/footprint.sh @@ -62,10 +62,10 @@ doit() fi { - scripts/config.pl unset MBEDTLS_NET_C || true - scripts/config.pl unset MBEDTLS_TIMING_C || true - scripts/config.pl unset MBEDTLS_FS_IO || true - scripts/config.pl --force set MBEDTLS_NO_PLATFORM_ENTROPY || true + scripts/config.py unset MBEDTLS_NET_C || true + scripts/config.py unset MBEDTLS_TIMING_C || true + scripts/config.py unset MBEDTLS_FS_IO || true + scripts/config.py --force set MBEDTLS_NO_PLATFORM_ENTROPY || true } >/dev/null 2>&1 make clean >/dev/null diff --git a/scripts/memory.sh b/scripts/memory.sh index 3dad2899c..c415f92d5 100755 --- a/scripts/memory.sh +++ b/scripts/memory.sh @@ -46,10 +46,10 @@ do_config() echo "" echo "config-$NAME:" cp configs/config-$NAME.h $CONFIG_H - scripts/config.pl unset MBEDTLS_SSL_SRV_C + scripts/config.py unset MBEDTLS_SSL_SRV_C for FLAG in $UNSET_LIST; do - scripts/config.pl unset $FLAG + scripts/config.py unset $FLAG done grep -F SSL_MAX_CONTENT_LEN $CONFIG_H || echo 'SSL_MAX_CONTENT_LEN=16384' diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index c7bf4281d..7f7c4bdbd 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -622,7 +622,7 @@ component_test_large_ecdsa_key_signature () { SMALL_MPI_MAX_SIZE=136 # Small enough to interfere with the EC signatures msg "build: cmake + MBEDTLS_MPI_MAX_SIZE=${SMALL_MPI_MAX_SIZE}, gcc, ASan" # ~ 1 min 50s - scripts/config.pl set MBEDTLS_MPI_MAX_SIZE $SMALL_MPI_MAX_SIZE + scripts/config.py set MBEDTLS_MPI_MAX_SIZE $SMALL_MPI_MAX_SIZE CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -662,7 +662,7 @@ component_test_default_cmake_gcc_asan () { component_test_full_cmake_gcc_asan () { msg "build: full config, cmake, gcc, ASan" - scripts/config.pl full + scripts/config.py full CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -684,7 +684,7 @@ component_test_ref_configs () { component_test_sslv3 () { msg "build: Default + SSLv3 (ASan build)" # ~ 6 min - scripts/config.pl set MBEDTLS_SSL_PROTO_SSL3 + scripts/config.py set MBEDTLS_SSL_PROTO_SSL3 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -701,7 +701,7 @@ component_test_sslv3 () { component_test_no_renegotiation () { msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min - scripts/config.pl unset MBEDTLS_SSL_RENEGOTIATION + scripts/config.py unset MBEDTLS_SSL_RENEGOTIATION CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -714,8 +714,8 @@ component_test_no_renegotiation () { component_test_no_pem_no_fs () { msg "build: Default + !MBEDTLS_PEM_PARSE_C + !MBEDTLS_FS_IO (ASan build)" - scripts/config.pl unset MBEDTLS_PEM_PARSE_C - scripts/config.pl unset MBEDTLS_FS_IO + scripts/config.py unset MBEDTLS_PEM_PARSE_C + scripts/config.py unset MBEDTLS_FS_IO CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -728,7 +728,7 @@ component_test_no_pem_no_fs () { component_test_rsa_no_crt () { msg "build: Default + RSA_NO_CRT (ASan build)" # ~ 6 min - scripts/config.pl set MBEDTLS_RSA_NO_CRT + scripts/config.py set MBEDTLS_RSA_NO_CRT CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -744,8 +744,8 @@ component_test_rsa_no_crt () { component_test_everest () { msg "build: Everest ECDH context (ASan build)" # ~ 6 min - scripts/config.pl unset MBEDTLS_ECDH_LEGACY_CONTEXT - scripts/config.pl set MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED + scripts/config.py unset MBEDTLS_ECDH_LEGACY_CONTEXT + scripts/config.py set MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED CC=clang cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -762,8 +762,8 @@ component_test_everest () { component_test_small_ssl_out_content_len () { msg "build: small SSL_OUT_CONTENT_LEN (ASan build)" - scripts/config.pl set MBEDTLS_SSL_IN_CONTENT_LEN 16384 - scripts/config.pl set MBEDTLS_SSL_OUT_CONTENT_LEN 4096 + scripts/config.py set MBEDTLS_SSL_IN_CONTENT_LEN 16384 + scripts/config.py set MBEDTLS_SSL_OUT_CONTENT_LEN 4096 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -773,8 +773,8 @@ component_test_small_ssl_out_content_len () { component_test_small_ssl_in_content_len () { msg "build: small SSL_IN_CONTENT_LEN (ASan build)" - scripts/config.pl set MBEDTLS_SSL_IN_CONTENT_LEN 4096 - scripts/config.pl set MBEDTLS_SSL_OUT_CONTENT_LEN 16384 + scripts/config.py set MBEDTLS_SSL_IN_CONTENT_LEN 4096 + scripts/config.py set MBEDTLS_SSL_OUT_CONTENT_LEN 16384 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -784,7 +784,7 @@ component_test_small_ssl_in_content_len () { component_test_small_ssl_dtls_max_buffering () { msg "build: small MBEDTLS_SSL_DTLS_MAX_BUFFERING #0" - scripts/config.pl set MBEDTLS_SSL_DTLS_MAX_BUFFERING 1000 + scripts/config.py set MBEDTLS_SSL_DTLS_MAX_BUFFERING 1000 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -794,7 +794,7 @@ component_test_small_ssl_dtls_max_buffering () { component_test_small_mbedtls_ssl_dtls_max_buffering () { msg "build: small MBEDTLS_SSL_DTLS_MAX_BUFFERING #1" - scripts/config.pl set MBEDTLS_SSL_DTLS_MAX_BUFFERING 190 + scripts/config.py set MBEDTLS_SSL_DTLS_MAX_BUFFERING 190 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -804,7 +804,7 @@ component_test_small_mbedtls_ssl_dtls_max_buffering () { component_test_full_cmake_clang () { msg "build: cmake, full config, clang" # ~ 50s - scripts/config.pl full + scripts/config.py full CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check -D ENABLE_TESTING=On . make @@ -823,8 +823,8 @@ component_test_full_cmake_clang () { component_build_deprecated () { msg "build: make, full config + DEPRECATED_WARNING, gcc -O" # ~ 30s - scripts/config.pl full - scripts/config.pl set MBEDTLS_DEPRECATED_WARNING + scripts/config.py full + scripts/config.py set MBEDTLS_DEPRECATED_WARNING # Build with -O -Wextra to catch a maximum of issues. make CC=gcc CFLAGS='-O -Werror -Wall -Wextra' lib programs make CC=gcc CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests @@ -832,8 +832,8 @@ component_build_deprecated () { msg "build: make, full config + DEPRECATED_REMOVED, clang -O" # ~ 30s # No cleanup, just tweak the configuration and rebuild make clean - scripts/config.pl unset MBEDTLS_DEPRECATED_WARNING - scripts/config.pl set MBEDTLS_DEPRECATED_REMOVED + scripts/config.py unset MBEDTLS_DEPRECATED_WARNING + scripts/config.py set MBEDTLS_DEPRECATED_REMOVED # Build with -O -Wextra to catch a maximum of issues. make CC=clang CFLAGS='-O -Werror -Wall -Wextra' lib programs make CC=clang CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests @@ -873,12 +873,12 @@ component_build_default_make_gcc_and_cxx () { component_test_no_use_psa_crypto_full_cmake_asan() { # full minus MBEDTLS_USE_PSA_CRYPTO: run the same set of tests as basic-build-test.sh msg "build: cmake, full config minus MBEDTLS_USE_PSA_CRYPTO, ASan" - scripts/config.pl full - scripts/config.pl set MBEDTLS_ECP_RESTARTABLE # not using PSA, so enable restartable ECC - scripts/config.pl unset MBEDTLS_PSA_CRYPTO_C - scripts/config.pl unset MBEDTLS_USE_PSA_CRYPTO - scripts/config.pl unset MBEDTLS_PSA_ITS_FILE_C - scripts/config.pl unset MBEDTLS_PSA_CRYPTO_STORAGE_C + scripts/config.py full + scripts/config.py set MBEDTLS_ECP_RESTARTABLE # not using PSA, so enable restartable ECC + scripts/config.py unset MBEDTLS_PSA_CRYPTO_C + scripts/config.py unset MBEDTLS_USE_PSA_CRYPTO + scripts/config.py unset MBEDTLS_PSA_ITS_FILE_C + scripts/config.py unset MBEDTLS_PSA_CRYPTO_STORAGE_C CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -903,9 +903,9 @@ component_test_no_use_psa_crypto_full_cmake_asan() { component_test_check_params_functionality () { msg "build+test: MBEDTLS_CHECK_PARAMS functionality" - scripts/config.pl full # includes CHECK_PARAMS + scripts/config.py full # includes CHECK_PARAMS # Make MBEDTLS_PARAM_FAILED call mbedtls_param_failed(). - scripts/config.pl unset MBEDTLS_CHECK_PARAMS_ASSERT + scripts/config.py unset MBEDTLS_CHECK_PARAMS_ASSERT # Only build and run tests. Do not build sample programs, because # they don't have a mbedtls_param_failed() function. make CC=gcc CFLAGS='-Werror -O1' lib test @@ -913,22 +913,22 @@ component_test_check_params_functionality () { component_test_check_params_without_platform () { msg "build+test: MBEDTLS_CHECK_PARAMS without MBEDTLS_PLATFORM_C" - scripts/config.pl full # includes CHECK_PARAMS + scripts/config.py full # includes CHECK_PARAMS # Keep MBEDTLS_PARAM_FAILED as assert. - scripts/config.pl unset MBEDTLS_PLATFORM_EXIT_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_FPRINTF_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_MEMORY - scripts/config.pl unset MBEDTLS_PLATFORM_PRINTF_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_SNPRINTF_ALT - scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED - scripts/config.pl unset MBEDTLS_PLATFORM_C + scripts/config.py unset MBEDTLS_PLATFORM_EXIT_ALT + scripts/config.py unset MBEDTLS_PLATFORM_TIME_ALT + scripts/config.py unset MBEDTLS_PLATFORM_FPRINTF_ALT + scripts/config.py unset MBEDTLS_PLATFORM_MEMORY + scripts/config.py unset MBEDTLS_PLATFORM_PRINTF_ALT + scripts/config.py unset MBEDTLS_PLATFORM_SNPRINTF_ALT + scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.py unset MBEDTLS_PLATFORM_C make CC=gcc CFLAGS='-Werror -O1' all test } component_test_check_params_silent () { msg "build+test: MBEDTLS_CHECK_PARAMS with alternative MBEDTLS_PARAM_FAILED()" - scripts/config.pl full # includes CHECK_PARAMS + scripts/config.py full # includes CHECK_PARAMS # Set MBEDTLS_PARAM_FAILED to nothing. sed -i 's/.*\(#define MBEDTLS_PARAM_FAILED( cond )\).*/\1/' "$CONFIG_H" make CC=gcc CFLAGS='-Werror -O1' all test @@ -939,19 +939,19 @@ component_test_no_platform () { # This should catch missing mbedtls_printf definitions, and by disabling file # IO, it should catch missing '#include ' msg "build: full config except platform/fsio/net, make, gcc, C99" # ~ 30s - scripts/config.pl full - scripts/config.pl unset MBEDTLS_PLATFORM_C - scripts/config.pl unset MBEDTLS_NET_C - scripts/config.pl unset MBEDTLS_PLATFORM_MEMORY - scripts/config.pl unset MBEDTLS_PLATFORM_PRINTF_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_FPRINTF_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_SNPRINTF_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_TIME_ALT - scripts/config.pl unset MBEDTLS_PLATFORM_EXIT_ALT - scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED - scripts/config.pl unset MBEDTLS_FS_IO - scripts/config.pl unset MBEDTLS_PSA_CRYPTO_STORAGE_C - scripts/config.pl unset MBEDTLS_PSA_ITS_FILE_C + scripts/config.py full + scripts/config.py unset MBEDTLS_PLATFORM_C + scripts/config.py unset MBEDTLS_NET_C + scripts/config.py unset MBEDTLS_PLATFORM_MEMORY + scripts/config.py unset MBEDTLS_PLATFORM_PRINTF_ALT + scripts/config.py unset MBEDTLS_PLATFORM_FPRINTF_ALT + scripts/config.py unset MBEDTLS_PLATFORM_SNPRINTF_ALT + scripts/config.py unset MBEDTLS_PLATFORM_TIME_ALT + scripts/config.py unset MBEDTLS_PLATFORM_EXIT_ALT + scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.py unset MBEDTLS_FS_IO + scripts/config.py unset MBEDTLS_PSA_CRYPTO_STORAGE_C + scripts/config.py unset MBEDTLS_PSA_ITS_FILE_C # Note, _DEFAULT_SOURCE needs to be defined for platforms using glibc version >2.19, # to re-enable platform integration features otherwise disabled in C99 builds make CC=gcc CFLAGS='-Werror -Wall -Wextra -std=c99 -pedantic -O0 -D_DEFAULT_SOURCE' lib programs @@ -961,23 +961,23 @@ component_test_no_platform () { component_build_no_std_function () { # catch compile bugs in _uninit functions msg "build: full config with NO_STD_FUNCTION, make, gcc" # ~ 30s - scripts/config.pl full - scripts/config.pl set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS - scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.py full + scripts/config.py set MBEDTLS_PLATFORM_NO_STD_FUNCTIONS + scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' } component_build_no_ssl_srv () { msg "build: full config except ssl_srv.c, make, gcc" # ~ 30s - scripts/config.pl full - scripts/config.pl unset MBEDTLS_SSL_SRV_C + scripts/config.py full + scripts/config.py unset MBEDTLS_SSL_SRV_C make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' } component_build_no_ssl_cli () { msg "build: full config except ssl_cli.c, make, gcc" # ~ 30s - scripts/config.pl full - scripts/config.pl unset MBEDTLS_SSL_CLI_C + scripts/config.py full + scripts/config.py unset MBEDTLS_SSL_CLI_C make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0' } @@ -985,18 +985,18 @@ component_build_no_sockets () { # Note, C99 compliance can also be tested with the sockets support disabled, # as that requires a POSIX platform (which isn't the same as C99). msg "build: full config except net_sockets.c, make, gcc -std=c99 -pedantic" # ~ 30s - scripts/config.pl full - scripts/config.pl unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc. - scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux + scripts/config.py full + scripts/config.py unset MBEDTLS_NET_C # getaddrinfo() undeclared, etc. + scripts/config.py set MBEDTLS_NO_PLATFORM_ENTROPY # uses syscall() on GNU/Linux make CC=gcc CFLAGS='-Werror -Wall -Wextra -O0 -std=c99 -pedantic' lib } component_test_memory_buffer_allocator_backtrace () { msg "build: default config with memory buffer allocator and backtrace enabled" - scripts/config.pl set MBEDTLS_MEMORY_BUFFER_ALLOC_C - scripts/config.pl set MBEDTLS_PLATFORM_MEMORY - scripts/config.pl set MBEDTLS_MEMORY_BACKTRACE - scripts/config.pl set MBEDTLS_MEMORY_DEBUG + scripts/config.py set MBEDTLS_MEMORY_BUFFER_ALLOC_C + scripts/config.py set MBEDTLS_PLATFORM_MEMORY + scripts/config.py set MBEDTLS_MEMORY_BACKTRACE + scripts/config.py set MBEDTLS_MEMORY_DEBUG CC=gcc cmake . make @@ -1006,8 +1006,8 @@ component_test_memory_buffer_allocator_backtrace () { component_test_memory_buffer_allocator () { msg "build: default config with memory buffer allocator" - scripts/config.pl set MBEDTLS_MEMORY_BUFFER_ALLOC_C - scripts/config.pl set MBEDTLS_PLATFORM_MEMORY + scripts/config.py set MBEDTLS_MEMORY_BUFFER_ALLOC_C + scripts/config.py set MBEDTLS_PLATFORM_MEMORY CC=gcc cmake . make @@ -1022,7 +1022,7 @@ component_test_memory_buffer_allocator () { component_test_no_max_fragment_length () { # Run max fragment length tests with MFL disabled msg "build: default config except MFL extension (ASan build)" # ~ 30s - scripts/config.pl unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH + scripts/config.py unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -1032,7 +1032,7 @@ component_test_no_max_fragment_length () { component_test_asan_remove_peer_certificate () { msg "build: default config with MBEDTLS_SSL_KEEP_PEER_CERTIFICATE disabled (ASan build)" - scripts/config.pl unset MBEDTLS_SSL_KEEP_PEER_CERTIFICATE + scripts/config.py unset MBEDTLS_SSL_KEEP_PEER_CERTIFICATE CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -1048,9 +1048,9 @@ component_test_asan_remove_peer_certificate () { component_test_no_max_fragment_length_small_ssl_out_content_len () { msg "build: no MFL extension, small SSL_OUT_CONTENT_LEN (ASan build)" - scripts/config.pl unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH - scripts/config.pl set MBEDTLS_SSL_IN_CONTENT_LEN 16384 - scripts/config.pl set MBEDTLS_SSL_OUT_CONTENT_LEN 4096 + scripts/config.py unset MBEDTLS_SSL_MAX_FRAGMENT_LENGTH + scripts/config.py set MBEDTLS_SSL_IN_CONTENT_LEN 16384 + scripts/config.py set MBEDTLS_SSL_OUT_CONTENT_LEN 4096 CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -1060,9 +1060,9 @@ component_test_no_max_fragment_length_small_ssl_out_content_len () { component_test_when_no_ciphersuites_have_mac () { msg "build: when no ciphersuites have MAC" - scripts/config.pl unset MBEDTLS_CIPHER_NULL_CIPHER - scripts/config.pl unset MBEDTLS_ARC4_C - scripts/config.pl unset MBEDTLS_CIPHER_MODE_CBC + scripts/config.py unset MBEDTLS_CIPHER_NULL_CIPHER + scripts/config.py unset MBEDTLS_ARC4_C + scripts/config.py unset MBEDTLS_CIPHER_MODE_CBC make msg "test: !MBEDTLS_SSL_SOME_MODES_USE_MAC" @@ -1074,12 +1074,12 @@ component_test_when_no_ciphersuites_have_mac () { component_test_null_entropy () { msg "build: default config with MBEDTLS_TEST_NULL_ENTROPY (ASan build)" - scripts/config.pl set MBEDTLS_TEST_NULL_ENTROPY - scripts/config.pl set MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES - scripts/config.pl set MBEDTLS_ENTROPY_C - scripts/config.pl unset MBEDTLS_ENTROPY_NV_SEED - scripts/config.pl unset MBEDTLS_ENTROPY_HARDWARE_ALT - scripts/config.pl unset MBEDTLS_HAVEGE_C + scripts/config.py set MBEDTLS_TEST_NULL_ENTROPY + scripts/config.py set MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES + scripts/config.py set MBEDTLS_ENTROPY_C + scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED + scripts/config.py unset MBEDTLS_ENTROPY_HARDWARE_ALT + scripts/config.py unset MBEDTLS_HAVEGE_C CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan -D UNSAFE_BUILD=ON . make @@ -1089,9 +1089,9 @@ component_test_null_entropy () { component_test_platform_calloc_macro () { msg "build: MBEDTLS_PLATFORM_{CALLOC/FREE}_MACRO enabled (ASan build)" - scripts/config.pl set MBEDTLS_PLATFORM_MEMORY - scripts/config.pl set MBEDTLS_PLATFORM_CALLOC_MACRO calloc - scripts/config.pl set MBEDTLS_PLATFORM_FREE_MACRO free + scripts/config.py set MBEDTLS_PLATFORM_MEMORY + scripts/config.py set MBEDTLS_PLATFORM_CALLOC_MACRO calloc + scripts/config.py set MBEDTLS_PLATFORM_FREE_MACRO free CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . make @@ -1117,7 +1117,7 @@ component_build_mbedtls_config_file () { msg "build: make with MBEDTLS_CONFIG_FILE" # ~40s # Use the full config so as to catch a maximum of places where # the check of MBEDTLS_CONFIG_FILE might be missing. - scripts/config.pl full + scripts/config.py full sed 's!"check_config.h"!"mbedtls/check_config.h"!' <"$CONFIG_H" >full_config.h echo '#error "MBEDTLS_CONFIG_FILE is not working"' >"$CONFIG_H" make CFLAGS="-I '$PWD' -DMBEDTLS_CONFIG_FILE='\"full_config.h\"'" @@ -1127,7 +1127,7 @@ component_build_mbedtls_config_file () { component_test_m32_o0 () { # Build once with -O0, to compile out the i386 specific inline assembly msg "build: i386, make, gcc -O0 (ASan build)" # ~ 30s - scripts/config.pl full + scripts/config.py full make CC=gcc CFLAGS='-O0 -Werror -Wall -Wextra -m32 -fsanitize=address' LDFLAGS='-m32 -fsanitize=address' msg "test: i386, make, gcc -O0 (ASan build)" @@ -1143,7 +1143,7 @@ support_test_m32_o0 () { component_test_m32_o1 () { # Build again with -O1, to compile in the i386 specific inline assembly msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s - scripts/config.pl full + scripts/config.py full make CC=gcc CFLAGS='-O1 -Werror -Wall -Wextra -m32 -fsanitize=address' LDFLAGS='-m32 -fsanitize=address' msg "test: i386, make, gcc -O1 (ASan build)" @@ -1158,8 +1158,8 @@ support_test_m32_o1 () { component_test_m32_everest () { msg "build: i386, Everest ECDH context (ASan build)" # ~ 6 min - scripts/config.pl unset MBEDTLS_ECDH_LEGACY_CONTEXT - scripts/config.pl set MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED + scripts/config.py unset MBEDTLS_ECDH_LEGACY_CONTEXT + scripts/config.py set MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED make CC=gcc CFLAGS='-O2 -Werror -Wall -Wextra -m32 -fsanitize=address' LDFLAGS='-m32 -fsanitize=address' msg "test: i386, Everest ECDH context - main suites (inc. selftests) (ASan build)" # ~ 50s @@ -1178,7 +1178,7 @@ support_test_m32_everest () { component_test_mx32 () { msg "build: 64-bit ILP32, make, gcc" # ~ 30s - scripts/config.pl full + scripts/config.py full make CC=gcc CFLAGS='-Werror -Wall -Wextra -mx32' LDFLAGS='-mx32' msg "test: 64-bit ILP32, make, gcc" @@ -1193,13 +1193,13 @@ support_test_mx32 () { component_build_arm_none_eabi_gcc () { msg "build: arm-none-eabi-gcc, make" # ~ 10s - scripts/config.pl baremetal + scripts/config.py baremetal make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib } component_build_arm_none_eabi_gcc_arm5vte () { msg "build: arm-none-eabi-gcc -march=arm5vte, make" # ~ 10s - scripts/config.pl baremetal + scripts/config.py baremetal # Build for a target platform that's close to what Debian uses # for its "armel" distribution (https://wiki.debian.org/ArmEabiPort). # See https://github.com/ARMmbed/mbedtls/pull/2169 and comments. @@ -1210,8 +1210,8 @@ component_build_arm_none_eabi_gcc_arm5vte () { component_build_arm_none_eabi_gcc_no_udbl_division () { msg "build: arm-none-eabi-gcc -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s - scripts/config.pl baremetal - scripts/config.pl set MBEDTLS_NO_UDBL_DIVISION + scripts/config.py baremetal + scripts/config.py set MBEDTLS_NO_UDBL_DIVISION make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib echo "Checking that software 64-bit division is not required" if_build_succeeded not grep __aeabi_uldiv library/*.o @@ -1219,8 +1219,8 @@ component_build_arm_none_eabi_gcc_no_udbl_division () { component_build_arm_none_eabi_gcc_no_64bit_multiplication () { msg "build: arm-none-eabi-gcc MBEDTLS_NO_64BIT_MULTIPLICATION, make" # ~ 10s - scripts/config.pl baremetal - scripts/config.pl set MBEDTLS_NO_64BIT_MULTIPLICATION + scripts/config.py baremetal + scripts/config.py set MBEDTLS_NO_64BIT_MULTIPLICATION make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -O1 -march=armv6-m -mthumb' lib echo "Checking that software 64-bit multiplication is not required" if_build_succeeded not grep __aeabi_lmul library/*.o @@ -1228,7 +1228,7 @@ component_build_arm_none_eabi_gcc_no_64bit_multiplication () { component_build_armcc () { msg "build: ARM Compiler 5, make" - scripts/config.pl baremetal + scripts/config.py baremetal make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib make clean @@ -1251,7 +1251,7 @@ component_build_armcc () { component_test_allow_sha1 () { msg "build: allow SHA1 in certificates by default" - scripts/config.pl set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES + scripts/config.py set MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES make CFLAGS='-Werror -Wall -Wextra' msg "test: allow SHA1 in certificates by default" make test @@ -1280,7 +1280,7 @@ support_build_mingw() { component_test_memsan () { msg "build: MSan (clang)" # ~ 1 min 20s - scripts/config.pl unset MBEDTLS_AESNI_C # memsan doesn't grok asm + scripts/config.py unset MBEDTLS_AESNI_C # memsan doesn't grok asm CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . make diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 4b71ff69b..6419f05e4 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -67,8 +67,8 @@ export CFLAGS=' --coverage -g3 -O0 ' export LDFLAGS=' --coverage' make clean cp "$CONFIG_H" "$CONFIG_BAK" -scripts/config.pl full -scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE +scripts/config.py full +scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE make -j diff --git a/tests/scripts/curves.pl b/tests/scripts/curves.pl index 4791d5521..3e2255277 100755 --- a/tests/scripts/curves.pl +++ b/tests/scripts/curves.pl @@ -46,13 +46,13 @@ for my $curve (@curves) { system( "make clean" ) and die; # depends on a specific curve. Also, ignore error if it wasn't enabled - system( "scripts/config.pl unset MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED" ); + system( "scripts/config.py unset MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED" ); print "\n******************************************\n"; print "* Testing without curve: $curve\n"; print "******************************************\n"; - system( "scripts/config.pl unset $curve" ) + system( "scripts/config.py unset $curve" ) and abort "Failed to disable $curve\n"; system( "CFLAGS='-Werror -Wall -Wextra' make lib" ) diff --git a/tests/scripts/depends-hashes.pl b/tests/scripts/depends-hashes.pl index f57e7ed88..92bcceb82 100755 --- a/tests/scripts/depends-hashes.pl +++ b/tests/scripts/depends-hashes.pl @@ -58,11 +58,11 @@ for my $hash (@hashes) { print "* Testing without hash: $hash\n"; print "******************************************\n"; - system( "scripts/config.pl unset $hash" ) + system( "scripts/config.py unset $hash" ) and abort "Failed to disable $hash\n"; for my $opt (@ssl) { - system( "scripts/config.pl unset $opt" ) + system( "scripts/config.py unset $opt" ) and abort "Failed to disable $opt\n"; } diff --git a/tests/scripts/depends-pkalgs.pl b/tests/scripts/depends-pkalgs.pl index 97a43e881..e3eac005d 100755 --- a/tests/scripts/depends-pkalgs.pl +++ b/tests/scripts/depends-pkalgs.pl @@ -73,10 +73,10 @@ while( my ($alg, $extras) = each %algs ) { print "* Testing without alg: $alg\n"; print "******************************************\n"; - system( "scripts/config.pl unset $alg" ) + system( "scripts/config.py unset $alg" ) and abort "Failed to disable $alg\n"; for my $opt (@$extras) { - system( "scripts/config.pl unset $opt" ) + system( "scripts/config.py unset $opt" ) and abort "Failed to disable $opt\n"; } diff --git a/tests/scripts/key-exchanges.pl b/tests/scripts/key-exchanges.pl index 3bf7ae34f..be029c7bd 100755 --- a/tests/scripts/key-exchanges.pl +++ b/tests/scripts/key-exchanges.pl @@ -47,10 +47,10 @@ for my $kex (@kexes) { print "******************************************\n"; # full config with all key exchanges disabled except one - system( "scripts/config.pl full" ) and abort "Failed config full\n"; + system( "scripts/config.py full" ) and abort "Failed config full\n"; for my $k (@kexes) { next if $k eq $kex; - system( "scripts/config.pl unset $k" ) + system( "scripts/config.py unset $k" ) and abort "Failed to disable $k\n"; } diff --git a/tests/scripts/list-symbols.sh b/tests/scripts/list-symbols.sh index 6ecc199bf..1c348a79c 100755 --- a/tests/scripts/list-symbols.sh +++ b/tests/scripts/list-symbols.sh @@ -13,7 +13,7 @@ if grep -i cmake Makefile >/dev/null; then fi cp include/mbedtls/config.h include/mbedtls/config.h.bak -scripts/config.pl full +scripts/config.py full make clean make_ret= CFLAGS=-fno-asynchronous-unwind-tables make lib \ diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 47b6b80c9..91fceeaec 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -300,9 +300,9 @@ requires_not_i686() { } # Calculate the input & output maximum content lengths set in the config -MAX_CONTENT_LEN=$( ../scripts/config.pl get MBEDTLS_SSL_MAX_CONTENT_LEN || echo "16384") -MAX_IN_LEN=$( ../scripts/config.pl get MBEDTLS_SSL_IN_CONTENT_LEN || echo "$MAX_CONTENT_LEN") -MAX_OUT_LEN=$( ../scripts/config.pl get MBEDTLS_SSL_OUT_CONTENT_LEN || echo "$MAX_CONTENT_LEN") +MAX_CONTENT_LEN=$( ../scripts/config.py get MBEDTLS_SSL_MAX_CONTENT_LEN || echo "16384") +MAX_IN_LEN=$( ../scripts/config.py get MBEDTLS_SSL_IN_CONTENT_LEN || echo "$MAX_CONTENT_LEN") +MAX_OUT_LEN=$( ../scripts/config.py get MBEDTLS_SSL_OUT_CONTENT_LEN || echo "$MAX_CONTENT_LEN") if [ "$MAX_IN_LEN" -lt "$MAX_CONTENT_LEN" ]; then MAX_CONTENT_LEN="$MAX_IN_LEN" @@ -3718,7 +3718,7 @@ run_test "Authentication: client no cert, ssl3" \ # default value (8) MAX_IM_CA='8' -MAX_IM_CA_CONFIG=$( ../scripts/config.pl get MBEDTLS_X509_MAX_INTERMEDIATE_CA) +MAX_IM_CA_CONFIG=$( ../scripts/config.py get MBEDTLS_X509_MAX_INTERMEDIATE_CA) if [ -n "$MAX_IM_CA_CONFIG" ] && [ "$MAX_IM_CA_CONFIG" -ne "$MAX_IM_CA" ]; then printf "The ${CONFIG_H} file contains a value for the configuration of\n" From bf359c7fc47a42fe3a95263e3d7f0fe006426b93 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 27 Jul 2019 23:56:04 +0200 Subject: [PATCH 08/28] Replace config.pl by a redirection to config.py Keep config.pl in Perl in case people are running "perl config.pl". --- scripts/config.pl | 318 +--------------------------------------------- 1 file changed, 4 insertions(+), 314 deletions(-) diff --git a/scripts/config.pl b/scripts/config.pl index b4b00581e..bd6c7e557 100755 --- a/scripts/config.pl +++ b/scripts/config.pl @@ -1,315 +1,5 @@ #!/usr/bin/env perl -# -# This file is part of mbed TLS (https://tls.mbed.org) -# -# Copyright (c) 2014-2016, ARM Limited, All Rights Reserved -# -# Purpose -# -# Comments and uncomments #define lines in the given header file and optionally -# sets their value or can get the value. This is to provide scripting control of -# what preprocessor symbols, and therefore what build time configuration flags -# are set in the 'config.h' file. -# -# Usage: config.pl [-f | --file ] [-o | --force] -# [set | unset | get | -# full | realfull] -# -# Full usage description provided below. -# -# The following options are disabled instead of enabled with "full". -# -# MBEDTLS_TEST_NULL_ENTROPY -# MBEDTLS_DEPRECATED_REMOVED -# MBEDTLS_HAVE_SSE2 -# MBEDTLS_PLATFORM_NO_STD_FUNCTIONS -# MBEDTLS_ECP_DP_M221_ENABLED -# MBEDTLS_ECP_DP_M383_ENABLED -# MBEDTLS_ECP_DP_M511_ENABLED -# MBEDTLS_MEMORY_BACKTRACE -# MBEDTLS_MEMORY_BUFFER_ALLOC_C -# MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES -# MBEDTLS_NO_PLATFORM_ENTROPY -# MBEDTLS_REMOVE_ARC4_CIPHERSUITES -# MBEDTLS_REMOVE_3DES_CIPHERSUITES -# MBEDTLS_SSL_HW_RECORD_ACCEL -# MBEDTLS_RSA_NO_CRT -# MBEDTLS_X509_ALLOW_EXTENSIONS_NON_V3 -# MBEDTLS_X509_ALLOW_UNSUPPORTED_CRITICAL_EXTENSION -# - this could be enabled if the respective tests were adapted -# MBEDTLS_ZLIB_SUPPORT -# MBEDTLS_PKCS11_C -# MBEDTLS_NO_UDBL_DIVISION -# MBEDTLS_NO_64BIT_MULTIPLICATION -# MBEDTLS_PSA_CRYPTO_SPM -# MBEDTLS_PSA_INJECT_ENTROPY -# MBEDTLS_ECP_RESTARTABLE -# and any symbol beginning _ALT -# - -use warnings; -use strict; - -my $config_file = "include/mbedtls/config.h"; -my $usage = < | --file ] [-o | --force] - [set | unset | get | - full | realfull | baremetal] - -Commands - set [] - Uncomments or adds a #define for the to - the configuration file, and optionally making it - of . - If the symbol isn't present in the file an error - is returned. - unset - Comments out the #define for the given symbol if - present in the configuration file. - get - Finds the #define for the given symbol, returning - an exitcode of 0 if the symbol is found, and 1 if - not. The value of the symbol is output if one is - specified in the configuration file. - full - Uncomments all #define's in the configuration file - excluding some reserved symbols, until the - 'Module configuration options' section - realfull - Uncomments all #define's with no exclusions - baremetal - Sets full configuration suitable for baremetal build. - -Options - -f | --file - The file or file path for the configuration file - to edit. When omitted, the following default is - used: - $config_file - -o | --force - If the symbol isn't present in the configuration - file when setting its value, a #define is - appended to the end of the file. - -EOU - -my @excluded = qw( -MBEDTLS_TEST_NULL_ENTROPY -MBEDTLS_DEPRECATED_REMOVED -MBEDTLS_HAVE_SSE2 -MBEDTLS_PLATFORM_NO_STD_FUNCTIONS -MBEDTLS_ECP_DP_M221_ENABLED -MBEDTLS_ECP_DP_M383_ENABLED -MBEDTLS_ECP_DP_M511_ENABLED -MBEDTLS_MEMORY_DEBUG -MBEDTLS_MEMORY_BACKTRACE -MBEDTLS_MEMORY_BUFFER_ALLOC_C -MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES -MBEDTLS_NO_PLATFORM_ENTROPY -MBEDTLS_RSA_NO_CRT -MBEDTLS_REMOVE_ARC4_CIPHERSUITES -MBEDTLS_REMOVE_3DES_CIPHERSUITES -MBEDTLS_SSL_HW_RECORD_ACCEL -MBEDTLS_X509_ALLOW_EXTENSIONS_NON_V3 -MBEDTLS_X509_ALLOW_UNSUPPORTED_CRITICAL_EXTENSION -MBEDTLS_ZLIB_SUPPORT -MBEDTLS_PKCS11_C -MBEDTLS_NO_UDBL_DIVISION -MBEDTLS_NO_64BIT_MULTIPLICATION -MBEDTLS_PSA_CRYPTO_SPM -MBEDTLS_PSA_INJECT_ENTROPY -MBEDTLS_ECP_RESTARTABLE -MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED -_ALT\s*$ -); - -# Things that should be disabled in "baremetal" -my @excluded_baremetal = qw( -MBEDTLS_NET_C -MBEDTLS_TIMING_C -MBEDTLS_FS_IO -MBEDTLS_ENTROPY_NV_SEED -MBEDTLS_HAVE_TIME -MBEDTLS_HAVE_TIME_DATE -MBEDTLS_DEPRECATED_WARNING -MBEDTLS_HAVEGE_C -MBEDTLS_THREADING_C -MBEDTLS_THREADING_PTHREAD -MBEDTLS_MEMORY_BACKTRACE -MBEDTLS_MEMORY_BUFFER_ALLOC_C -MBEDTLS_PLATFORM_TIME_ALT -MBEDTLS_PLATFORM_FPRINTF_ALT -MBEDTLS_PSA_ITS_FILE_C -MBEDTLS_PSA_CRYPTO_STORAGE_C -); - -# Things that should be enabled in "full" even if they match @excluded -my @non_excluded = qw( -PLATFORM_[A-Z0-9]+_ALT -); - -# Things that should be enabled in "baremetal" -my @non_excluded_baremetal = qw( -MBEDTLS_NO_PLATFORM_ENTROPY -); - -# Process the command line arguments - -my $force_option = 0; - -my ($arg, $name, $value, $action); - -while ($arg = shift) { - - # Check if the argument is an option - if ($arg eq "-f" || $arg eq "--file") { - $config_file = shift; - - -f $config_file or die "No such file: $config_file\n"; - - } - elsif ($arg eq "-o" || $arg eq "--force") { - $force_option = 1; - - } - else - { - # ...else assume it's a command - $action = $arg; - - if ($action eq "full" || $action eq "realfull" || $action eq "baremetal" ) { - # No additional parameters - die $usage if @ARGV; - - } - elsif ($action eq "unset" || $action eq "get") { - die $usage unless @ARGV; - $name = shift; - - } - elsif ($action eq "set") { - die $usage unless @ARGV; - $name = shift; - $value = shift if @ARGV; - - } - else { - die "Command '$action' not recognised.\n\n".$usage; - } - } -} - -# If no command was specified, exit... -if ( not defined($action) ){ die $usage; } - -# Check the config file is present -if (! -f $config_file) { - - chdir '..' or die; - - # Confirm this is the project root directory and try again - if ( !(-d 'scripts' && -d 'include' && -d 'library' && -f $config_file) ) { - die "If no file specified, must be run from the project root or scripts directory.\n"; - } -} - - -# Now read the file and process the contents - -open my $config_read, '<', $config_file or die "read $config_file: $!\n"; -my @config_lines = <$config_read>; -close $config_read; - -# Add required baremetal symbols to the list that is included. -if ( $action eq "baremetal" ) { - @non_excluded = ( @non_excluded, @non_excluded_baremetal ); -} - -my ($exclude_re, $no_exclude_re, $exclude_baremetal_re); -if ($action eq "realfull") { - $exclude_re = qr/^$/; - $no_exclude_re = qr/./; -} else { - $exclude_re = join '|', @excluded; - $no_exclude_re = join '|', @non_excluded; -} -if ( $action eq "baremetal" ) { - $exclude_baremetal_re = join '|', @excluded_baremetal; -} - -my $config_write = undef; -if ($action ne "get") { - open $config_write, '>', $config_file or die "write $config_file: $!\n"; -} - -my $done; -for my $line (@config_lines) { - if ($action eq "full" || $action eq "realfull" || $action eq "baremetal" ) { - if ($line =~ /name SECTION: Module configuration options/) { - $done = 1; - } - - if (!$done && $line =~ m!^//\s?#define! && - ( $line !~ /$exclude_re/ || $line =~ /$no_exclude_re/ ) && - ( $action ne "baremetal" || ( $line !~ /$exclude_baremetal_re/ ) ) ) { - $line =~ s!^//\s?!!; - } - if (!$done && $line =~ m!^\s?#define! && - ! ( ( $line !~ /$exclude_re/ || $line =~ /$no_exclude_re/ ) && - ( $action ne "baremetal" || ( $line !~ /$exclude_baremetal_re/ ) ) ) ) { - $line =~ s!^!//!; - } - } elsif ($action eq "unset") { - if (!$done && $line =~ /^\s*#define\s*$name\b/) { - $line = '//' . $line; - $done = 1; - } - } elsif (!$done && $action eq "set") { - if ($line =~ m!^(?://)?\s*#define\s*$name\b!) { - $line = "#define $name"; - $line .= " $value" if defined $value && $value ne ""; - $line .= "\n"; - $done = 1; - } - } elsif (!$done && $action eq "get") { - if ($line =~ /^\s*#define\s*$name(?:\s+(.*?))\s*(?:$|\/\*|\/\/)/) { - $value = $1; - $done = 1; - } - } - - if (defined $config_write) { - print $config_write $line or die "write $config_file: $!\n"; - } -} - -# Did the set command work? -if ($action eq "set" && $force_option && !$done) { - - # If the force option was set, append the symbol to the end of the file - my $line = "#define $name"; - $line .= " $value" if defined $value && $value ne ""; - $line .= "\n"; - $done = 1; - - print $config_write $line or die "write $config_file: $!\n"; -} - -if (defined $config_write) { - close $config_write or die "close $config_file: $!\n"; -} - -if ($action eq "get") { - if ($done) { - if ($value ne '') { - print "$value\n"; - } - exit 0; - } else { - # If the symbol was not found, return an error - exit 1; - } -} - -if ($action eq "full" && !$done) { - die "Configuration section was not found in $config_file\n"; - -} - -if ($action ne "full" && $action ne "unset" && !$done) { - die "A #define for the symbol $name was not found in $config_file\n"; -} - -__END__ +# Backward compatibility redirection +my $py = $0; +$py =~ s/\.pl$/.py/; +exec 'python3', $py, @ARGV From 90b30b618b8c8ae56f9d5a9ee2e219b1b359f4e8 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 28 Jul 2019 00:36:53 +0200 Subject: [PATCH 09/28] Print help when invoked with no arguments --- scripts/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/config.py b/scripts/config.py index 9fe2e516a..cf347e96f 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -374,7 +374,10 @@ if __name__ == '__main__': args = parser.parse_args() config = ConfigFile(args.file) - if args.command == 'get': + if args.command is None: + parser.print_help() + return 1 + elif args.command == 'get': if args.symbol in config: value = config[args.symbol] if value: From 0fa5efb7cea9ea1083f2d82bc9ab79b5c42cba7b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 28 Jul 2019 13:30:06 +0200 Subject: [PATCH 10/28] Fix encoding errors config.h is encoded in UTF-8. --- scripts/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index cf347e96f..3b2aa63a7 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -246,7 +246,7 @@ class ConfigFile(Config): super().__init__() self.filename = filename self.current_section = 'header' - with open(filename) as file: + with open(filename, 'r', encoding='utf-8') as file: self.templates = [self._parse_line(line) for line in file] self.current_section = None @@ -314,7 +314,7 @@ class ConfigFile(Config): """ if filename is None: filename = self.filename - with open(filename, 'w') as output: + with open(filename, 'w', encoding='utf-8') as output: self.write_to_stream(output) if __name__ == '__main__': From 98eb36557d53d89698391f49593a6b6e5b56fc99 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 28 Jul 2019 16:39:19 +0200 Subject: [PATCH 11/28] Fix 'config.py set' without --force The `set` command can act on any known symbol. --- scripts/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/config.py b/scripts/config.py index 3b2aa63a7..1a2b1595e 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -384,7 +384,7 @@ if __name__ == '__main__': sys.stdout.write(value + '\n') return args.symbol not in config elif args.command == 'set': - if not args.force and args.symbol not in config: + if not args.force and args.symbol not in config.settings: sys.stderr.write("A #define for the symbol {} " "was not found in {}" .format(args.symbol, args.file)) From 6cf3127527060f1d861dc9bf1666329c09aea631 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 29 Jul 2019 23:42:50 +0200 Subject: [PATCH 12/28] Report an error if switching to Python fails --- scripts/config.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/config.pl b/scripts/config.pl index bd6c7e557..4f6df09fd 100755 --- a/scripts/config.pl +++ b/scripts/config.pl @@ -2,4 +2,6 @@ # Backward compatibility redirection my $py = $0; $py =~ s/\.pl$/.py/; -exec 'python3', $py, @ARGV +exec 'python3', $py, @ARGV; +print STDERR "$0: python3: $!\n"; +exit 127; From 208e4ec5d3bb86addd524ffac6f63b2c5fe6c9ad Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 29 Jul 2019 23:43:20 +0200 Subject: [PATCH 13/28] Also search config.h near the script By default, this script looks for include/mbedtls/config.h relative to the current directory. This allows running config.py from outside the build tree. To support out-of-tree builds where config.h and config.py are in the source tree and the current directory is in the build tree, also try DIRECTORY_CONTAINING_SCRIPT/../include/mbedtls/config.h, and the equivalent with symbolic links traversed. --- scripts/config.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index 1a2b1595e..aaea48408 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -24,6 +24,7 @@ Basic usage, to read the Mbed TLS or Mbed Crypto configuration: ## ## This file is part of Mbed TLS (https://tls.mbed.org) +import os import re class Setting: @@ -237,12 +238,20 @@ class ConfigFile(Config): and modify the configuration. """ - default_path = 'include/mbedtls/config.h' + _path_in_tree = 'include/mbedtls/config.h' + default_path = [_path_in_tree, + os.path.join(os.path.dirname(__file__), + os.pardir, + _path_in_tree), + os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), + _path_in_tree)] def __init__(self, filename=None): """Read the Mbed TLS configuration file.""" if filename is None: - filename = self.default_path + for filename in self.default_path: + if os.path.lexists(filename): + break super().__init__() self.filename = filename self.current_section = 'header' From 55cc4dbb5c9b1823b23b5c914f2754c6a6c18d77 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Aug 2019 23:13:23 +0200 Subject: [PATCH 14/28] Fix Config.unset() making the name known --- scripts/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index aaea48408..81c35d0a8 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -131,9 +131,10 @@ class Config: def unset(self, name): """Make name unset (inactive). - name remains known. + name remains known if it was known before. """ - self.set(name) + if name not in self.settings: + return self.settings[name].active = False def adapt(self, adapter): From 435ce22920f0360a21f02c16e2dd5605d5c1c73c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Aug 2019 23:13:47 +0200 Subject: [PATCH 15/28] Fix --force requiring an argument --- scripts/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/config.py b/scripts/config.py index 81c35d0a8..382831acf 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -338,6 +338,7 @@ if __name__ == '__main__': Default: {}. """.format(ConfigFile.default_path)) parser.add_argument('--force', '-o', + action='store_true', help="""For the set command, if SYMBOL is not present, add a definition for it.""") parser.add_argument('--write', '-w', From 0c7fcd210ff3fb3fe4fa2065a415b4b7fde6b7a6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Aug 2019 23:14:00 +0200 Subject: [PATCH 16/28] Fix "--force set" without a value sneaking a None in --- scripts/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/config.py b/scripts/config.py index 382831acf..910594945 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -361,7 +361,8 @@ if __name__ == '__main__': found, unless --force is passed. """) parser_set.add_argument('symbol', metavar='SYMBOL') - parser_set.add_argument('value', metavar='VALUE', nargs='?') + parser_set.add_argument('value', metavar='VALUE', nargs='?', + default='') parser_unset = subparsers.add_parser('unset', help="""Comment out the #define for SYMBOL. Do nothing if none From 221df1e0ef2b8eee4caa9dde632ea4951e624ce9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Aug 2019 23:14:29 +0200 Subject: [PATCH 17/28] Fix "#define ... not found" error when using the default file name Also make that error message end with a newline. --- scripts/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index 910594945..d3836c77d 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -398,8 +398,8 @@ if __name__ == '__main__': elif args.command == 'set': if not args.force and args.symbol not in config.settings: sys.stderr.write("A #define for the symbol {} " - "was not found in {}" - .format(args.symbol, args.file)) + "was not found in {}\n" + .format(args.symbol, config.filename)) return 1 config.set(args.symbol, value=args.value) elif args.command == 'unset': From c190c90889b7b663d175fd65089ebc393e252758 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Aug 2019 23:31:05 +0200 Subject: [PATCH 18/28] Documentation improvements --- scripts/config.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index d3836c77d..58be3b289 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -53,12 +53,10 @@ class Config: if there is a #define for it whether commented out or not. This class supports the following protocols: - * `name in config` is True if the symbol `name` is set in the - configuration, False otherwise (whether `name` is known but commented - out or not known at all). - * `config[name]` is the value of the macro `name`. If `name` is not - set, raise `KeyError` (even if a definition for `name` is present - but commented out). + * `name in config` is `True` if the symbol `name` is active, `False` + otherwise (whether `name` is inactive or not known). + * `config[name]` is the value of the macro `name`. If `name` is inactive, + raise `KeyError` (even if `name` is known). * `config[name] = value` sets the value associated to `name`. `name` must be known, but does not need to be set. This does not cause name to become set. @@ -156,7 +154,7 @@ def is_full_section(section): return section.endswith('support') or section.endswith('modules') def realfull_adapter(_name, active, section): - """Uncomment everything in the system and feature sections.""" + """Activate all symbols found in the system and feature sections.""" if not is_full_section(section): return active return True @@ -300,7 +298,8 @@ class ConfigFile(Config): def _format_template(self, name, indent, middle): """Build a line for config.h for the given setting. - The line has the form "#define ". + The line has the form "#define " + where is "#define ". """ setting = self.settings[name] return ''.join([indent, @@ -341,7 +340,7 @@ if __name__ == '__main__': action='store_true', help="""For the set command, if SYMBOL is not present, add a definition for it.""") - parser.add_argument('--write', '-w', + parser.add_argument('--write', '-w', metavar='FILE', help="""File to write to instead of the input file.""") subparsers = parser.add_subparsers(dest='command', title='Commands') From 878acd6490048e46f5199d48bb4dcee7e52280ea Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Aug 2019 23:32:38 +0200 Subject: [PATCH 19/28] Test script for config.py Run config.py with various options and store the results in files. This script also supports the now-removed config.pl. This is a framework to run non-regression tests on config.py: run it with the old version, run it with the new version, and compare the output. This is deliberately not a functional test suite so that we don't need to maintain a set of known outputs. When something changes in config.py (or config.h), run the script before, run it after, and check manually whether any differences in the output are acceptable. --- tests/scripts/test_config_script.py | 177 ++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100755 tests/scripts/test_config_script.py diff --git a/tests/scripts/test_config_script.py b/tests/scripts/test_config_script.py new file mode 100755 index 000000000..0a93fa03e --- /dev/null +++ b/tests/scripts/test_config_script.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 + +"""Test helper for the Mbed TLS configuration file tool + +Run config.py with various parameters and write the results to files. + +This is a harness to help regression testing, not a functional tester. +Sample usage: + + test_config_script.py -d old + ## Modify config.py and/or config.h ## + test_config_script.py -d new + diff -ru old new +""" + +## Copyright (C) 2019, ARM Limited, All Rights Reserved +## SPDX-License-Identifier: Apache-2.0 +## +## Licensed under the Apache License, Version 2.0 (the "License"); you may +## not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +## This file is part of Mbed TLS (https://tls.mbed.org) + +import argparse +import glob +import os +import re +import shutil +import subprocess + +OUTPUT_FILE_PREFIX = 'config-' + +def output_file_name(directory, stem, extension): + return os.path.join(directory, + '{}{}.{}'.format(OUTPUT_FILE_PREFIX, + stem, extension)) + +def cleanup_directory(directory): + """Remove old output files.""" + for extension in []: + pattern = output_file_name(directory, '*', extension) + filenames = glob.glob(pattern) + for filename in filenames: + os.remove(filename) + +def prepare_directory(directory): + """Create the output directory if it doesn't exist yet. + + If there are old output files, remove them. + """ + if os.path.exists(directory): + cleanup_directory(directory) + else: + os.makedirs(directory) + +def guess_presets_from_help(help_text): + """Figure out what presets the script supports. + + help_text should be the output from running the script with --help. + """ + # Try the output format from config.py + hits = re.findall(r'\{([-\w,]+)\}', help_text) + for hit in hits: + words = set(hit.split(',')) + if 'get' in words and 'set' in words and 'unset' in words: + words.remove('get') + words.remove('set') + words.remove('unset') + return words + # Try the output format from config.pl + hits = re.findall(r'\n +([-\w]+) +- ', help_text) + if hits: + return hits + raise Exception("Unable to figure out supported presets. Pass the '-p' option.") + +def list_presets(options): + """Return the list of presets to test. + + The list is taken from the command line if present, otherwise it is + extracted from running the config script with --help. + """ + if options.presets: + return re.split(r'[ ,]+', options.presets) + else: + help_text = subprocess.run([options.script, '--help'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT).stdout + return guess_presets_from_help(help_text.decode('ascii')) + +def run_one(options, args): + """Run the config script with the given arguments. + + Write the following files: + * config-xxx.h: modified file. + * config-xxx.out: standard output. + * config-xxx.err: standard output. + * config-xxx.status: exit code. + """ + stem = '-'.join(args) + data_filename = output_file_name(options.output_directory, stem, 'h') + stdout_filename = output_file_name(options.output_directory, stem, 'out') + stderr_filename = output_file_name(options.output_directory, stem, 'err') + status_filename = output_file_name(options.output_directory, stem, 'status') + shutil.copy(options.input_file, data_filename) + # Pass only the file basename, not the full path, to avoid getting the + # directory name in error messages, which would make comparisons + # between output directories more difficult. + cmd = [os.path.abspath(options.script), + '-f', os.path.basename(data_filename)] + with open(stdout_filename, 'wb') as out: + with open(stderr_filename, 'wb') as err: + status = subprocess.call(cmd + args, + cwd=options.output_directory, + stdin=subprocess.DEVNULL, + stdout=out, stderr=err) + with open(status_filename, 'w') as status_file: + status_file.write('{}\n'.format(status)) + +### A list of symbols to test with set and unset. +TEST_SYMBOLS = [ + 'CUSTOM_OPTION', + 'DOES_NOT_EXIST', + 'MBEDTLS_AES_C', + 'MBEDTLS_NO_UDBL_DIVISION', + 'MBEDTLS_PLATFORM_ZEROIZE_ALT', +] + +### A list of symbols to test with set with a value. +TEST_SYMBOLS_WITH_VALUE = [ + 'CUSTOM_VALUE', + 'MBEDTLS_MPI_MAX_SIZE', +] + +def run_all(options): + """Run all the command lines to test.""" + presets = list_presets(options) + for preset in presets: + run_one(options, [preset]) + for symbol in TEST_SYMBOLS: + run_one(options, ['set', symbol]) + run_one(options, ['--force', 'set', symbol]) + run_one(options, ['unset', symbol]) + for symbol in TEST_SYMBOLS_WITH_VALUE: + run_one(options, ['set', symbol, 'value']) + run_one(options, ['--force', 'set', symbol, 'value']) + +def main(): + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-d', metavar='DIR', + dest='output_directory', required=True, + help="""Output directory.""") + parser.add_argument('-f', metavar='FILE', + dest='input_file', default='include/mbedtls/config.h', + help="""Config file (default: %(default)s).""") + parser.add_argument('-p', metavar='PRESET,...', + dest='presets', + help="""Presets to test (default: guessed from --help).""") + parser.add_argument('-s', metavar='FILE', + dest='script', default='scripts/config.py', + help="""Configuration script (default: %(default)s).""") + options = parser.parse_args() + prepare_directory(options.output_directory) + run_all(options) + +if __name__ == '__main__': + main() From 2fd7ffa81bac88bed9fdafee855fb113db9ef5e4 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 4 Sep 2019 22:10:34 +0200 Subject: [PATCH 20/28] cmake: fix Python requirement Perl is no longer needed. Python must be version 3. Version 2 is not suitable. The variable is PYTHONINTERP_FOUND, not PYTHON_FOUND. --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f2b502e7..4350d4e28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,9 +51,9 @@ set(CTR_DRBG_128_BIT_KEY_WARNING "${WARNING_BORDER}" "${CTR_DRBG_128_BIT_KEY_WARN_L3}" "${WARNING_BORDER}") -find_package(PythonInterp) -find_package(Perl) -if(PYTHON_FOUND) +# Python 3 is only needed here to check for configuration warnings. +find_package(PythonInterp 3) +if(PYTHONINTERP_FOUND) # If 128-bit keys are configured for CTR_DRBG, display an appropriate warning execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/config.py -f ${CMAKE_CURRENT_SOURCE_DIR}/include/mbedtls/config.h get MBEDTLS_CTR_DRBG_USE_128_BIT_KEY From 00ed2e19863016b6447ea5f08a5ccafddb2c39aa Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 4 Sep 2019 22:13:02 +0200 Subject: [PATCH 21/28] cmake: update interpreter requirement for the test suite generator The test suite generator has been a Python script for a long time, but tests/CMakeLists.txt still looked for Perl. The reference to PYTHON_INTERP only worked due to a call to find_package(PythonInterp) in the toplevel CMakeLists.txt, and cmake would not have printed the expected error message if python was not available. --- tests/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ecf6f34b2..d3d487a1c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,9 +10,9 @@ if(ENABLE_ZLIB_SUPPORT) set(libs ${libs} ${ZLIB_LIBRARIES}) endif(ENABLE_ZLIB_SUPPORT) -find_package(Perl) -if(NOT PERL_FOUND) - message(FATAL_ERROR "Cannot build test suites without Perl") +find_package(PythonInterp) +if(NOT PYTHONINTERP_FOUND) + message(FATAL_ERROR "Cannot build test suites without Python 2 or 3") endif() # Enable definition of various functions used throughout the testsuite From e3734bd13ab98ef4c18729522b43d8d0a82774ea Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 4 Sep 2019 22:51:33 +0200 Subject: [PATCH 22/28] Remove redundant test case --- tests/scripts/test_config_script.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/scripts/test_config_script.py b/tests/scripts/test_config_script.py index 0a93fa03e..a71b35792 100755 --- a/tests/scripts/test_config_script.py +++ b/tests/scripts/test_config_script.py @@ -128,7 +128,6 @@ def run_one(options, args): ### A list of symbols to test with set and unset. TEST_SYMBOLS = [ 'CUSTOM_OPTION', - 'DOES_NOT_EXIST', 'MBEDTLS_AES_C', 'MBEDTLS_NO_UDBL_DIVISION', 'MBEDTLS_PLATFORM_ZEROIZE_ALT', From f686042554c8bb37e5fd7a0b4b5ffda2fbc5edeb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 4 Sep 2019 22:51:47 +0200 Subject: [PATCH 23/28] Fix config.py output when a symbol has acquired or lost a value Normally a valueless symbol remains valueless and a symbol with a value keeps having one. But just in case a symbol does get changed from valueless to having a value, make sure there's a space between the symbol and the value. And if a symbol gets changed from having a value to valueless, strip trailing whitespace. Add corresponding tests. Also fix the case of a valueless symbol added with the set method, which would have resulted in attempting to use None as a string. This only happened with the Python API, not with the command line API. --- scripts/config.py | 14 +++++++++++++- tests/scripts/test_config_script.py | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/config.py b/scripts/config.py index 58be3b289..e01b9d541 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -302,10 +302,22 @@ class ConfigFile(Config): where is "#define ". """ setting = self.settings[name] + value = setting.value + if value is None: + value = '' + # Normally the whitespace to separte the symbol name from the + # value is part of middle, and there's no whitespace for a symbol + # with no value. But if a symbol has been changed from having a + # value to not having one, the whitespace is wrong, so fix it. + if value: + if middle[-1] not in '\t ': + middle += ' ' + else: + middle = middle.rstrip() return ''.join([indent, '' if setting.active else '//', middle, - setting.value]).rstrip() + value]).rstrip() def write_to_stream(self, output): """Write the whole configuration to output.""" diff --git a/tests/scripts/test_config_script.py b/tests/scripts/test_config_script.py index a71b35792..dd3ecbbdb 100755 --- a/tests/scripts/test_config_script.py +++ b/tests/scripts/test_config_script.py @@ -129,6 +129,7 @@ def run_one(options, args): TEST_SYMBOLS = [ 'CUSTOM_OPTION', 'MBEDTLS_AES_C', + 'MBEDTLS_MPI_MAX_SIZE', 'MBEDTLS_NO_UDBL_DIVISION', 'MBEDTLS_PLATFORM_ZEROIZE_ALT', ] @@ -136,6 +137,7 @@ TEST_SYMBOLS = [ ### A list of symbols to test with set with a value. TEST_SYMBOLS_WITH_VALUE = [ 'CUSTOM_VALUE', + 'MBEDTLS_AES_C', 'MBEDTLS_MPI_MAX_SIZE', ] @@ -151,6 +153,7 @@ def run_all(options): for symbol in TEST_SYMBOLS_WITH_VALUE: run_one(options, ['set', symbol, 'value']) run_one(options, ['--force', 'set', symbol, 'value']) + run_one(options, ['unset', symbol]) def main(): """Command line entry point.""" From a103c180327481af3f4e2c736adb783e97e34f33 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 5 Sep 2019 20:29:22 +0200 Subject: [PATCH 24/28] Compatibility redirect: if python3 is not available, try python --- scripts/config.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/config.pl b/scripts/config.pl index 4f6df09fd..ed6727639 100755 --- a/scripts/config.pl +++ b/scripts/config.pl @@ -4,4 +4,6 @@ my $py = $0; $py =~ s/\.pl$/.py/; exec 'python3', $py, @ARGV; print STDERR "$0: python3: $!\n"; +exec 'python', $py, @ARGV; +print STDERR "$0: python: $!\n"; exit 127; From 0409bcdef0dd9998838f06e2c17a6ed36201d899 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 13 Sep 2019 15:14:42 +0200 Subject: [PATCH 25/28] Compatibility redirect: add copyright notice --- scripts/config.pl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/config.pl b/scripts/config.pl index ed6727639..95e31913a 100755 --- a/scripts/config.pl +++ b/scripts/config.pl @@ -1,5 +1,23 @@ #!/usr/bin/env perl # Backward compatibility redirection + +## Copyright (C) 2019, ARM Limited, All Rights Reserved +## SPDX-License-Identifier: Apache-2.0 +## +## Licensed under the Apache License, Version 2.0 (the "License"); you may +## not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +## This file is part of Mbed TLS (https://tls.mbed.org) + my $py = $0; $py =~ s/\.pl$/.py/; exec 'python3', $py, @ARGV; From 61695e70f8af738fdf8fd2171b53413489cd1144 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 13 Sep 2019 15:17:01 +0200 Subject: [PATCH 26/28] config.py testing: also test the get command --- tests/scripts/test_config_script.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/scripts/test_config_script.py b/tests/scripts/test_config_script.py index dd3ecbbdb..87c3d4695 100755 --- a/tests/scripts/test_config_script.py +++ b/tests/scripts/test_config_script.py @@ -147,10 +147,12 @@ def run_all(options): for preset in presets: run_one(options, [preset]) for symbol in TEST_SYMBOLS: + run_one(options, ['get', symbol]) run_one(options, ['set', symbol]) run_one(options, ['--force', 'set', symbol]) run_one(options, ['unset', symbol]) for symbol in TEST_SYMBOLS_WITH_VALUE: + run_one(options, ['get', symbol]) run_one(options, ['set', symbol, 'value']) run_one(options, ['--force', 'set', symbol, 'value']) run_one(options, ['unset', symbol]) From fd7ad33ee941789086718eae9fe65d9a3651631b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 19 Sep 2019 12:18:23 +0200 Subject: [PATCH 27/28] Consolidate tests for set with/without values We currently test setting a symbol with a value even if it didn't originally had one and vice versa. So there's no need to have separate lists of symbols to test with. Just test everything we want to test with each symbol. --- tests/scripts/test_config_script.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tests/scripts/test_config_script.py b/tests/scripts/test_config_script.py index 87c3d4695..45f15a164 100755 --- a/tests/scripts/test_config_script.py +++ b/tests/scripts/test_config_script.py @@ -125,20 +125,17 @@ def run_one(options, args): with open(status_filename, 'w') as status_file: status_file.write('{}\n'.format(status)) -### A list of symbols to test with set and unset. +### A list of symbols to test with. +### This script currently tests what happens when you change a symbol from +### having a value to not having a value or vice versa. This is not +### necessarily useful behavior, and we may not consider it a bug if +### config.py stops handling that case correctly. TEST_SYMBOLS = [ - 'CUSTOM_OPTION', - 'MBEDTLS_AES_C', - 'MBEDTLS_MPI_MAX_SIZE', - 'MBEDTLS_NO_UDBL_DIVISION', - 'MBEDTLS_PLATFORM_ZEROIZE_ALT', -] - -### A list of symbols to test with set with a value. -TEST_SYMBOLS_WITH_VALUE = [ - 'CUSTOM_VALUE', - 'MBEDTLS_AES_C', - 'MBEDTLS_MPI_MAX_SIZE', + 'CUSTOM_SYMBOL', # does not exist + 'MBEDTLS_AES_C', # set, no value + 'MBEDTLS_MPI_MAX_SIZE', # unset, has a value + 'MBEDTLS_NO_UDBL_DIVISION', # unset, in "System support" + 'MBEDTLS_PLATFORM_ZEROIZE_ALT', # unset, in "Customisation configuration options" ] def run_all(options): @@ -150,9 +147,6 @@ def run_all(options): run_one(options, ['get', symbol]) run_one(options, ['set', symbol]) run_one(options, ['--force', 'set', symbol]) - run_one(options, ['unset', symbol]) - for symbol in TEST_SYMBOLS_WITH_VALUE: - run_one(options, ['get', symbol]) run_one(options, ['set', symbol, 'value']) run_one(options, ['--force', 'set', symbol, 'value']) run_one(options, ['unset', symbol]) From 16a25e005d53fc19095891e143a3bbbd020ef8e4 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 19 Sep 2019 12:19:24 +0200 Subject: [PATCH 28/28] Add set+get tests The tests were not covering get for a symbol with a value. No symbol has an uncommented value in the default config.h. (Actually there's _CRT_SECURE_NO_DEPRECATE, but that's a bit of a hack that this script is not expected to handle, so don't use it). Add tests of "get FOO" after "set FOO" and "set FOO value", so that we have coverage for "get FOO" when "FOO" has a value. --- tests/scripts/test_config_script.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/scripts/test_config_script.py b/tests/scripts/test_config_script.py index 45f15a164..40ed9fd9b 100755 --- a/tests/scripts/test_config_script.py +++ b/tests/scripts/test_config_script.py @@ -96,21 +96,30 @@ def list_presets(options): stderr=subprocess.STDOUT).stdout return guess_presets_from_help(help_text.decode('ascii')) -def run_one(options, args): +def run_one(options, args, stem_prefix='', input_file=None): """Run the config script with the given arguments. - Write the following files: + Take the original content from input_file if specified, defaulting + to options.input_file if input_file is None. + + Write the following files, where xxx contains stem_prefix followed by + a filename-friendly encoding of args: * config-xxx.h: modified file. * config-xxx.out: standard output. * config-xxx.err: standard output. * config-xxx.status: exit code. + + Return ("xxx+", "path/to/config-xxx.h") which can be used as + stem_prefix and input_file to call this function again with new args. """ - stem = '-'.join(args) + if input_file is None: + input_file = options.input_file + stem = stem_prefix + '-'.join(args) data_filename = output_file_name(options.output_directory, stem, 'h') stdout_filename = output_file_name(options.output_directory, stem, 'out') stderr_filename = output_file_name(options.output_directory, stem, 'err') status_filename = output_file_name(options.output_directory, stem, 'status') - shutil.copy(options.input_file, data_filename) + shutil.copy(input_file, data_filename) # Pass only the file basename, not the full path, to avoid getting the # directory name in error messages, which would make comparisons # between output directories more difficult. @@ -124,6 +133,7 @@ def run_one(options, args): stdout=out, stderr=err) with open(status_filename, 'w') as status_file: status_file.write('{}\n'.format(status)) + return stem + "+", data_filename ### A list of symbols to test with. ### This script currently tests what happens when you change a symbol from @@ -145,9 +155,11 @@ def run_all(options): run_one(options, [preset]) for symbol in TEST_SYMBOLS: run_one(options, ['get', symbol]) - run_one(options, ['set', symbol]) + (stem, filename) = run_one(options, ['set', symbol]) + run_one(options, ['get', symbol], stem_prefix=stem, input_file=filename) run_one(options, ['--force', 'set', symbol]) - run_one(options, ['set', symbol, 'value']) + (stem, filename) = run_one(options, ['set', symbol, 'value']) + run_one(options, ['get', symbol], stem_prefix=stem, input_file=filename) run_one(options, ['--force', 'set', symbol, 'value']) run_one(options, ['unset', symbol])