Merge pull request #8177 from gilles-peskine-arm/generated-files-off-in-release

Generated files off in release

CMakeLists.txt

@@ -57,7 +57,10 @@ option(ENABLE_PROGRAMS "Build Mbed TLS programs." ON)
 option(UNSAFE_BUILD "Allow unsafe builds. These builds ARE NOT SECURE." OFF)
 option(MBEDTLS_FATAL_WARNINGS "Compiler warnings treated as errors" ON)
 if(CMAKE_HOST_WIN32)
-    option(GEN_FILES "Generate the auto-generated files as needed" OFF)
+    # N.B. The comment on the next line is significant! If you change it,
+    # edit the sed command in prepare_release.sh that modifies
+    # CMakeLists.txt.
+    option(GEN_FILES "Generate the auto-generated files as needed" OFF) # off in development
 else()
     option(GEN_FILES "Generate the auto-generated files as needed" ON)
 endif()

Makefile

@@ -36,6 +36,29 @@ generated_files: programs/generated_files
 generated_files: tests/generated_files
 generated_files: visualc_files
 
+# Set GEN_FILES to the empty string to disable dependencies on generated
+# source files. Then `make generated_files` will only build files that
+# are missing, it will not rebuilt files that are present but out of date.
+# This is useful, for example, if you have a source tree where
+# `make generated_files` has already run and file timestamps reflect the
+# time the files were copied or extracted, and you are now in an environment
+# that lacks some of the necessary tools to re-generate the files.
+# If $(GEN_FILES) is non-empty, the generated source files' dependencies
+# are treated ordinarily, based on file timestamps.
+GEN_FILES ?= yes
+
+# In dependencies where the target is a configuration-independent generated
+# file, use `TARGET: $(gen_file_dep) DEPENDENCY1 DEPENDENCY2 ...`
+# rather than directly `TARGET: DEPENDENCY1 DEPENDENCY2 ...`. This
+# enables the re-generation to be turned off when GEN_FILES is disabled.
+ifdef GEN_FILES
+gen_file_dep =
+else
+# Order-only dependency: generate the target if it's absent, but don't
+# re-generate it if it's present but older than its dependencies.
+gen_file_dep = |
+endif
+
 .PHONY: visualc_files
 VISUALC_FILES = visualc/VS2013/mbedTLS.sln visualc/VS2013/mbedTLS.vcxproj
 # TODO: $(app).vcxproj for each $(app) in programs/
@@ -45,10 +68,10 @@ visualc_files: $(VISUALC_FILES)
 # present before it runs. It doesn't matter if the files aren't up-to-date,
 # they just need to be present.
 $(VISUALC_FILES): | library/generated_files
-$(VISUALC_FILES): scripts/generate_visualc_files.pl
-$(VISUALC_FILES): scripts/data_files/vs2013-app-template.vcxproj
-$(VISUALC_FILES): scripts/data_files/vs2013-main-template.vcxproj
-$(VISUALC_FILES): scripts/data_files/vs2013-sln-template.sln
+$(VISUALC_FILES): $(gen_file_dep) scripts/generate_visualc_files.pl
+$(VISUALC_FILES): $(gen_file_dep) scripts/data_files/vs2013-app-template.vcxproj
+$(VISUALC_FILES): $(gen_file_dep) scripts/data_files/vs2013-main-template.vcxproj
+$(VISUALC_FILES): $(gen_file_dep) scripts/data_files/vs2013-sln-template.sln
 # TODO: also the list of .c and .h source files, but not their content
 $(VISUALC_FILES):
 	echo "  Gen   $@ ..."

library/Makefile

@@ -315,21 +315,29 @@ GENERATED_FILES = \
         psa_crypto_driver_wrappers.c
 generated_files: $(GENERATED_FILES)
 
-error.c: ../scripts/generate_errors.pl
-error.c: ../scripts/data_files/error.fmt
-error.c: $(filter-out %config%,$(wildcard ../include/mbedtls/*.h))
+# See root Makefile
+GEN_FILES ?= yes
+ifdef GEN_FILES
+gen_file_dep =
+else
+gen_file_dep = |
+endif
+
+error.c: $(gen_file_dep) ../scripts/generate_errors.pl
+error.c: $(gen_file_dep) ../scripts/data_files/error.fmt
+error.c: $(gen_file_dep) $(filter-out %config%,$(wildcard ../include/mbedtls/*.h))
 error.c:
 	echo "  Gen   $@"
 	$(PERL) ../scripts/generate_errors.pl
 
-ssl_debug_helpers_generated.c: ../scripts/generate_ssl_debug_helpers.py
-ssl_debug_helpers_generated.c: $(filter-out %config%,$(wildcard ../include/mbedtls/*.h))
+ssl_debug_helpers_generated.c: $(gen_file_dep) ../scripts/generate_ssl_debug_helpers.py
+ssl_debug_helpers_generated.c: $(gen_file_dep) $(filter-out %config%,$(wildcard ../include/mbedtls/*.h))
 ssl_debug_helpers_generated.c:
 	echo "  Gen   $@"
 	$(PYTHON) ../scripts/generate_ssl_debug_helpers.py --mbedtls-root .. .
 
-version_features.c: ../scripts/generate_features.pl
-version_features.c: ../scripts/data_files/version_features.fmt
+version_features.c: $(gen_file_dep) ../scripts/generate_features.pl
+version_features.c: $(gen_file_dep) ../scripts/data_files/version_features.fmt
 ## The generated file only depends on the options that are present in mbedtls_config.h,
 ## not on which options are set. To avoid regenerating this file all the time
 ## when switching between configurations, don't declare mbedtls_config.h as a
@@ -340,8 +348,8 @@ version_features.c:
 	echo "  Gen   $@"
 	$(PERL) ../scripts/generate_features.pl
 
-psa_crypto_driver_wrappers.c: ../scripts/generate_driver_wrappers.py
-psa_crypto_driver_wrappers.c: ../scripts/data_files/driver_templates/psa_crypto_driver_wrappers.c.jinja
+psa_crypto_driver_wrappers.c: $(gen_file_dep) ../scripts/generate_driver_wrappers.py
+psa_crypto_driver_wrappers.c: $(gen_file_dep) ../scripts/data_files/driver_templates/psa_crypto_driver_wrappers.c.jinja
 psa_crypto_driver_wrappers.c:
 	echo "  Gen   $@"
 	$(PYTHON) ../scripts/generate_driver_wrappers.py

programs/Makefile

@@ -176,22 +176,30 @@ ${MBEDTLS_TEST_OBJS}:
 GENERATED_FILES = psa/psa_constant_names_generated.c test/query_config.c
 generated_files: $(GENERATED_FILES)
 
-psa/psa_constant_names_generated.c: ../scripts/generate_psa_constants.py
-psa/psa_constant_names_generated.c: ../include/psa/crypto_values.h
-psa/psa_constant_names_generated.c: ../include/psa/crypto_extra.h
-psa/psa_constant_names_generated.c: ../tests/suites/test_suite_psa_crypto_metadata.data
+# See root Makefile
+GEN_FILES ?= yes
+ifdef GEN_FILES
+gen_file_dep =
+else
+gen_file_dep = |
+endif
+
+psa/psa_constant_names_generated.c: $(gen_file_dep) ../scripts/generate_psa_constants.py
+psa/psa_constant_names_generated.c: $(gen_file_dep) ../include/psa/crypto_values.h
+psa/psa_constant_names_generated.c: $(gen_file_dep) ../include/psa/crypto_extra.h
+psa/psa_constant_names_generated.c: $(gen_file_dep) ../tests/suites/test_suite_psa_crypto_metadata.data
 psa/psa_constant_names_generated.c:
 	echo "  Gen   $@"
 	$(PYTHON) ../scripts/generate_psa_constants.py
 
-test/query_config.c: ../scripts/generate_query_config.pl
+test/query_config.c: $(gen_file_dep) ../scripts/generate_query_config.pl
 ## The generated file only depends on the options that are present in mbedtls_config.h,
 ## not on which options are set. To avoid regenerating this file all the time
 ## when switching between configurations, don't declare mbedtls_config.h as a
 ## dependency. Remove this file from your working tree if you've just added or
 ## removed an option in mbedtls_config.h.
-#test/query_config.c: ../include/mbedtls/mbedtls_config.h
-test/query_config.c: ../scripts/data_files/query_config.fmt
+#test/query_config.c: $(gen_file_dep) ../include/mbedtls/mbedtls_config.h
+test/query_config.c: $(gen_file_dep) ../scripts/data_files/query_config.fmt
 test/query_config.c:
 	echo "  Gen   $@"
 	$(PERL) ../scripts/generate_query_config.pl

scripts/prepare_release.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
-#
+
+print_usage()
+{
+    cat <<EOF
+Usage: $0 [OPTION]...
+Prepare the source tree for a release.
+
+Options:
+  -u    Prepare for development (undo the release preparation)
+EOF
+}
+
 # Copyright The Mbed TLS Contributors
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -14,52 +25,32 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Purpose
-#
-# For adapting gitignore files for releases so generated files can be included.
-#
-# Usage: gitignore_add_generated_files.sh  [ -h | --help ] etc
-#
 
 set -eu
 
-print_usage()
-{
-    echo "Usage: $0"
-    echo -e "  -h|--help\t\tPrint this help."
-    echo -e "  -i|--ignore\t\tAdd generated files to the gitignores."
-    echo -e "  -u|--unignore\t\tRemove generated files from the gitignores."
-}
-
-if [[ $# -eq 0 ]]; then
+if [ $# -ne 0 ] && [ "$1" = "--help" ]; then
     print_usage
-    exit 1
-elif [[ $# -ge 2 ]]; then
-    echo "Too many arguments!"
-    exit 1
+    exit
 fi
 
-case "$1" in
-    -i | --ignore)
-        IGNORE=true
-        ;;
-    -u | --uignore)
-        IGNORE=false
-        ;;
-    -h | --help | "")
-        print_usage
-        exit 1
-        ;;
-    *)
-        echo "Unknown argument: $1"
-        echo "run '$0 --help' for options"
-        exit 1
-esac
+unrelease= # if non-empty, we're in undo-release mode
+while getopts u OPTLET; do
+    case $OPTLET in
+        u) unrelease=1;;
+        \?)
+            echo 1>&2 "$0: unknown option: -$OPTLET"
+            echo 1>&2 "Try '$0 --help' for more information."
+            exit 3;;
+    esac
+done
+
+
+
+#### .gitignore processing ####
 
 GITIGNORES=$(find . -name ".gitignore")
 for GITIGNORE in $GITIGNORES; do
-    if $IGNORE; then
+    if [ -n "$unrelease" ]; then
         sed -i '/###START_COMMENTED_GENERATED_FILES###/,/###END_COMMENTED_GENERATED_FILES###/s/^# //' $GITIGNORE
         sed -i 's/###START_COMMENTED_GENERATED_FILES###/###START_GENERATED_FILES###/' $GITIGNORE
         sed -i 's/###END_COMMENTED_GENERATED_FILES###/###END_GENERATED_FILES###/' $GITIGNORE
@@ -69,3 +60,23 @@ for GITIGNORE in $GITIGNORES; do
         sed -i 's/###END_GENERATED_FILES###/###END_COMMENTED_GENERATED_FILES###/' $GITIGNORE
     fi
 done
+
+
+
+#### Build scripts ####
+
+# GEN_FILES defaults on (non-empty) in development, off (empty) in releases
+if [ -n "$unrelease" ]; then
+    r=' yes'
+else
+    r=''
+fi
+sed -i 's/^\(GEN_FILES[ ?:]*=\)\([^#]*\)/\1'"$r/" Makefile */Makefile
+
+# GEN_FILES defaults on in development, off in releases
+if [ -n "$unrelease" ]; then
+    r='ON'
+else
+    r='OFF'
+fi
+sed -i '/[Oo][Ff][Ff] in development/! s/^\( *option *( *GEN_FILES  *"[^"]*"  *\)\([A-Za-z0-9][A-Za-z0-9]*\)/\1'"$r/" CMakeLists.txt

tests/Makefile

@@ -65,6 +65,14 @@ else
 PYTHON ?= $(shell if type python3 >/dev/null 2>/dev/null; then echo python3; else echo python; fi)
 endif
 
+# See root Makefile
+GEN_FILES ?= yes
+ifdef GEN_FILES
+gen_file_dep =
+else
+gen_file_dep = |
+endif
+
 .PHONY: generated_files
 GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \
 	$(PYTHON) scripts/generate_bignum_tests.py --list || \
@@ -97,7 +105,7 @@ generated_files: $(GENERATED_FILES)
 # Use an intermediate phony dependency so that parallel builds don't run
 # a separate instance of the recipe for each output file.
 .SECONDARY: generated_bignum_test_data generated_ecp_test_data generated_psa_test_data
-$(GENERATED_BIGNUM_DATA_FILES): generated_bignum_test_data
+$(GENERATED_BIGNUM_DATA_FILES): $(gen_file_dep) generated_bignum_test_data
 generated_bignum_test_data: scripts/generate_bignum_tests.py
 generated_bignum_test_data: ../scripts/mbedtls_dev/bignum_common.py
 generated_bignum_test_data: ../scripts/mbedtls_dev/bignum_core.py
@@ -109,7 +117,7 @@ generated_bignum_test_data:
 	echo "  Gen   $(GENERATED_BIGNUM_DATA_FILES)"
 	$(PYTHON) scripts/generate_bignum_tests.py
 
-$(GENERATED_ECP_DATA_FILES): generated_ecp_test_data
+$(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data
 generated_ecp_test_data: scripts/generate_ecp_tests.py
 generated_ecp_test_data: ../scripts/mbedtls_dev/bignum_common.py
 generated_ecp_test_data: ../scripts/mbedtls_dev/ecp.py
@@ -119,7 +127,7 @@ generated_ecp_test_data:
 	echo "  Gen   $(GENERATED_ECP_DATA_FILES)"
 	$(PYTHON) scripts/generate_ecp_tests.py
 
-$(GENERATED_PSA_DATA_FILES): generated_psa_test_data
+$(GENERATED_PSA_DATA_FILES): $(gen_file_dep) generated_psa_test_data
 generated_psa_test_data: scripts/generate_psa_tests.py
 generated_psa_test_data: ../scripts/mbedtls_dev/crypto_data_tests.py
 generated_psa_test_data: ../scripts/mbedtls_dev/crypto_knowledge.py