Merge pull request #3126 from gilles-peskine-arm/merge-crypto-stragglers-20200325
Merge mbed-crypto stragglers 2020-03-25
This commit is contained in:
Manuel Pégourié-Gonnard
GitHub
commit
2451ac698d
3 changed files with 342 additions and 0 deletions
21
ChangeLog.d/README
Normal file
21
ChangeLog.d/README
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
This directory contains changelog entries that have not yet been merged
|
||||
to the changelog file (../ChangeLog.md).
|
||||
|
||||
A changelog entry file must have the extension *.md and must have the
|
||||
following format:
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
### Section title
|
||||
|
||||
* Change descritpion.
|
||||
* Another change description.
|
||||
|
||||
### Another section title
|
||||
|
||||
* Yet another change description.
|
||||
* Yet again another change description.
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
See STANDARD_SECTIONS in ../scripts/assemble_changelog.py for
|
||||
recognized section titles.
|
||||
319
scripts/assemble_changelog.py
Executable file
319
scripts/assemble_changelog.py
Executable file
|
|
@ -0,0 +1,319 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""Assemble Mbed Crypto change log entries into the change log file.
|
||||
|
||||
Add changelog entries to the first level-2 section.
|
||||
Create a new level-2 section for unreleased changes if needed.
|
||||
Remove the input files unless --keep-entries is specified.
|
||||
"""
|
||||
|
||||
# Copyright (C) 2019, Arm Limited, All Rights Reserved
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# This file is part of Mbed Crypto (https://tls.mbed.org)
|
||||
|
||||
import argparse
|
||||
from collections import OrderedDict
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
class InputFormatError(Exception):
|
||||
def __init__(self, filename, line_number, message, *args, **kwargs):
|
||||
message = '{}:{}: {}'.format(filename, line_number,
|
||||
message.format(*args, **kwargs))
|
||||
super().__init__(message)
|
||||
|
||||
class LostContent(Exception):
|
||||
def __init__(self, filename, line):
|
||||
message = ('Lost content from {}: "{}"'.format(filename, line))
|
||||
super().__init__(message)
|
||||
|
||||
STANDARD_SECTIONS = (
|
||||
b'Interface changes',
|
||||
b'Default behavior changes',
|
||||
b'Requirement changes',
|
||||
b'New deprecations',
|
||||
b'Removals',
|
||||
b'New features',
|
||||
b'Security',
|
||||
b'Bug fixes',
|
||||
b'Performance improvements',
|
||||
b'Other changes',
|
||||
)
|
||||
|
||||
class ChangeLog:
|
||||
"""An Mbed Crypto changelog.
|
||||
|
||||
A changelog is a file in Markdown format. Each level 2 section title
|
||||
starts a version, and versions are sorted in reverse chronological
|
||||
order. Lines with a level 2 section title must start with '##'.
|
||||
|
||||
Within a version, there are multiple sections, each devoted to a kind
|
||||
of change: bug fix, feature request, etc. Section titles should match
|
||||
entries in STANDARD_SECTIONS exactly.
|
||||
|
||||
Within each section, each separate change should be on a line starting
|
||||
with a '*' bullet. There may be blank lines surrounding titles, but
|
||||
there should not be any blank line inside a section.
|
||||
"""
|
||||
|
||||
_title_re = re.compile(br'#*')
|
||||
def title_level(self, line):
|
||||
"""Determine whether the line is a title.
|
||||
|
||||
Return (level, content) where level is the Markdown section level
|
||||
(1 for '#', 2 for '##', etc.) and content is the section title
|
||||
without leading or trailing whitespace. For a non-title line,
|
||||
the level is 0.
|
||||
"""
|
||||
level = re.match(self._title_re, line).end()
|
||||
return level, line[level:].strip()
|
||||
|
||||
# Only accept dotted version numbers (e.g. "3.1", not "3").
|
||||
# Refuse ".x" in a version number where x is a letter: this indicates
|
||||
# a version that is not yet released. Something like "3.1a" is accepted.
|
||||
_version_number_re = re.compile(br'[0-9]+\.[0-9A-Za-z.]+')
|
||||
_incomplete_version_number_re = re.compile(br'.*\.[A-Za-z]')
|
||||
|
||||
def section_is_released_version(self, title):
|
||||
"""Whether this section is for a released version.
|
||||
|
||||
True if the given level-2 section title indicates that this section
|
||||
contains released changes, otherwise False.
|
||||
"""
|
||||
# Assume that a released version has a numerical version number
|
||||
# that follows a particular pattern. These criteria may be revised
|
||||
# as needed in future versions of this script.
|
||||
version_number = re.search(self._version_number_re, title)
|
||||
if version_number:
|
||||
return not re.search(self._incomplete_version_number_re,
|
||||
version_number.group(0))
|
||||
else:
|
||||
return False
|
||||
|
||||
def unreleased_version_title(self):
|
||||
"""The title to use if creating a new section for an unreleased version."""
|
||||
# pylint: disable=no-self-use; this method may be overridden
|
||||
return b'Unreleased changes'
|
||||
|
||||
def __init__(self, input_stream):
|
||||
"""Create a changelog object.
|
||||
|
||||
Populate the changelog object from the content of the file
|
||||
input_stream. This is typically a file opened for reading, but
|
||||
can be any generator returning the lines to read.
|
||||
"""
|
||||
# Content before the level-2 section where the new entries are to be
|
||||
# added.
|
||||
self.header = []
|
||||
# Content of the level-3 sections of where the new entries are to
|
||||
# be added.
|
||||
self.section_content = OrderedDict()
|
||||
for section in STANDARD_SECTIONS:
|
||||
self.section_content[section] = []
|
||||
# Content of level-2 sections for already-released versions.
|
||||
self.trailer = []
|
||||
self.read_main_file(input_stream)
|
||||
|
||||
def read_main_file(self, input_stream):
|
||||
"""Populate the changelog object from the content of the file.
|
||||
|
||||
This method is only intended to be called as part of the constructor
|
||||
of the class and may not act sensibly on an object that is already
|
||||
partially populated.
|
||||
"""
|
||||
# Parse the first level-2 section, containing changelog entries
|
||||
# for unreleased changes.
|
||||
# If we'll be expanding this section, everything before the first
|
||||
# level-3 section title ("###...") following the first level-2
|
||||
# section title ("##...") is passed through as the header
|
||||
# and everything after the second level-2 section title is passed
|
||||
# through as the trailer. Inside the first level-2 section,
|
||||
# split out the level-3 sections.
|
||||
# If we'll be creating a new version, the header is everything
|
||||
# before the point where we want to add the level-2 section
|
||||
# for this version, and the trailer is what follows.
|
||||
level_2_seen = 0
|
||||
current_section = None
|
||||
for line in input_stream:
|
||||
level, content = self.title_level(line)
|
||||
if level == 2:
|
||||
level_2_seen += 1
|
||||
if level_2_seen == 1:
|
||||
if self.section_is_released_version(content):
|
||||
self.header.append(b'## ' +
|
||||
self.unreleased_version_title() +
|
||||
b'\n\n')
|
||||
level_2_seen = 2
|
||||
elif level == 3 and level_2_seen == 1:
|
||||
current_section = content
|
||||
self.section_content.setdefault(content, [])
|
||||
if level_2_seen == 1 and current_section is not None:
|
||||
if level != 3 and line.strip():
|
||||
self.section_content[current_section].append(line)
|
||||
elif level_2_seen <= 1:
|
||||
self.header.append(line)
|
||||
else:
|
||||
self.trailer.append(line)
|
||||
|
||||
def add_file(self, input_stream):
|
||||
"""Add changelog entries from a file.
|
||||
|
||||
Read lines from input_stream, which is typically a file opened
|
||||
for reading. These lines must contain a series of level 3
|
||||
Markdown sections with recognized titles. The corresponding
|
||||
content is injected into the respective sections in the changelog.
|
||||
The section titles must be either one of the hard-coded values
|
||||
in STANDARD_SECTIONS in assemble_changelog.py or already present
|
||||
in ChangeLog.md. Section titles must match byte-for-byte except that
|
||||
leading or trailing whitespace is ignored.
|
||||
"""
|
||||
filename = input_stream.name
|
||||
current_section = None
|
||||
for line_number, line in enumerate(input_stream, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
level, content = self.title_level(line)
|
||||
if level == 3:
|
||||
current_section = content
|
||||
if current_section not in self.section_content:
|
||||
raise InputFormatError(filename, line_number,
|
||||
'Section {} is not recognized',
|
||||
str(current_section)[1:])
|
||||
elif level == 0:
|
||||
if current_section is None:
|
||||
raise InputFormatError(filename, line_number,
|
||||
'Missing section title at the beginning of the file')
|
||||
self.section_content[current_section].append(line)
|
||||
else:
|
||||
raise InputFormatError(filename, line_number,
|
||||
'Only level 3 headers (###) are permitted')
|
||||
|
||||
def write(self, filename):
|
||||
"""Write the changelog to the specified file.
|
||||
"""
|
||||
with open(filename, 'wb') as out:
|
||||
for line in self.header:
|
||||
out.write(line)
|
||||
for section, lines in self.section_content.items():
|
||||
if not lines:
|
||||
continue
|
||||
out.write(b'### ' + section + b'\n\n')
|
||||
for line in lines:
|
||||
out.write(line)
|
||||
out.write(b'\n')
|
||||
for line in self.trailer:
|
||||
out.write(line)
|
||||
|
||||
def check_output(generated_output_file, main_input_file, merged_files):
|
||||
"""Make sanity checks on the generated output.
|
||||
|
||||
The intent of these sanity checks is to have reasonable confidence
|
||||
that no content has been lost.
|
||||
|
||||
The sanity check is that every line that is present in an input file
|
||||
is also present in an output file. This is not perfect but good enough
|
||||
for now.
|
||||
"""
|
||||
generated_output = set(open(generated_output_file, 'rb'))
|
||||
for line in open(main_input_file, 'rb'):
|
||||
if line not in generated_output:
|
||||
raise LostContent('original file', line)
|
||||
for merged_file in merged_files:
|
||||
for line in open(merged_file, 'rb'):
|
||||
if line not in generated_output:
|
||||
raise LostContent(merged_file, line)
|
||||
|
||||
def finish_output(changelog, output_file, input_file, merged_files):
|
||||
"""Write the changelog to the output file.
|
||||
|
||||
The input file and the list of merged files are used only for sanity
|
||||
checks on the output.
|
||||
"""
|
||||
if os.path.exists(output_file) and not os.path.isfile(output_file):
|
||||
# The output is a non-regular file (e.g. pipe). Write to it directly.
|
||||
output_temp = output_file
|
||||
else:
|
||||
# The output is a regular file. Write to a temporary file,
|
||||
# then move it into place atomically.
|
||||
output_temp = output_file + '.tmp'
|
||||
changelog.write(output_temp)
|
||||
check_output(output_temp, input_file, merged_files)
|
||||
if output_temp != output_file:
|
||||
os.rename(output_temp, output_file)
|
||||
|
||||
def remove_merged_entries(files_to_remove):
|
||||
for filename in files_to_remove:
|
||||
os.remove(filename)
|
||||
|
||||
def merge_entries(options):
|
||||
"""Merge changelog entries into the changelog file.
|
||||
|
||||
Read the changelog file from options.input.
|
||||
Read entries to merge from the directory options.dir.
|
||||
Write the new changelog to options.output.
|
||||
Remove the merged entries if options.keep_entries is false.
|
||||
"""
|
||||
with open(options.input, 'rb') as input_file:
|
||||
changelog = ChangeLog(input_file)
|
||||
files_to_merge = glob.glob(os.path.join(options.dir, '*.md'))
|
||||
if not files_to_merge:
|
||||
sys.stderr.write('There are no pending changelog entries.\n')
|
||||
return
|
||||
for filename in files_to_merge:
|
||||
with open(filename, 'rb') as input_file:
|
||||
changelog.add_file(input_file)
|
||||
finish_output(changelog, options.output, options.input, files_to_merge)
|
||||
if not options.keep_entries:
|
||||
remove_merged_entries(files_to_merge)
|
||||
|
||||
def set_defaults(options):
|
||||
"""Add default values for missing options."""
|
||||
output_file = getattr(options, 'output', None)
|
||||
if output_file is None:
|
||||
options.output = options.input
|
||||
if getattr(options, 'keep_entries', None) is None:
|
||||
options.keep_entries = (output_file is not None)
|
||||
|
||||
def main():
|
||||
"""Command line entry point."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--dir', '-d', metavar='DIR',
|
||||
default='ChangeLog.d',
|
||||
help='Directory to read entries from'
|
||||
' (default: ChangeLog.d)')
|
||||
parser.add_argument('--input', '-i', metavar='FILE',
|
||||
default='ChangeLog.md',
|
||||
help='Existing changelog file to read from and augment'
|
||||
' (default: ChangeLog.md)')
|
||||
parser.add_argument('--keep-entries',
|
||||
action='store_true', dest='keep_entries', default=None,
|
||||
help='Keep the files containing entries'
|
||||
' (default: remove them if --output/-o is not specified)')
|
||||
parser.add_argument('--no-keep-entries',
|
||||
action='store_false', dest='keep_entries',
|
||||
help='Remove the files containing entries after they are merged'
|
||||
' (default: remove them if --output/-o is not specified)')
|
||||
parser.add_argument('--output', '-o', metavar='FILE',
|
||||
help='Output changelog file'
|
||||
' (default: overwrite the input)')
|
||||
options = parser.parse_args()
|
||||
set_defaults(options)
|
||||
merge_entries(options)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -5297,6 +5297,8 @@ void generate_random( int bytes_arg )
|
|||
size_t i;
|
||||
unsigned run;
|
||||
|
||||
TEST_ASSERT( bytes_arg >= 0 );
|
||||
|
||||
ASSERT_ALLOC( output, bytes + sizeof( trail ) );
|
||||
ASSERT_ALLOC( changed, bytes );
|
||||
memcpy( output + bytes, trail, sizeof( trail ) );
|
||||
|
|
|
|||
Loading…
Reference in a new issue