Build from submodule by default (make, cmake)

Adapt tests in all.sh:
- tests with submodule enabled (default) no longer need to enable it
  explicitly, and no longer need runtime tests, as those are now handled by
all other test cases in this script
- tests with submodule disabled (old default) now need to disable it
  explicitly, and execute some runtime tests, as those are no longer tested
anywhere else in this script

Adapt documentation in Readme: remove the section "building with submodule"
and replace it with a new section before the other building sections.
Purposefully don't document how to build not from the submodule, as that
option is going away soon.

CMakeLists.txt

@@ -9,7 +9,7 @@ option(USE_PKCS11_HELPER_LIBRARY "Build mbed TLS with the pkcs11-helper library.
 option(ENABLE_ZLIB_SUPPORT "Build mbed TLS with zlib library." OFF)
 
 option(ENABLE_PROGRAMS "Build mbed TLS programs." ON)
-option(USE_CRYPTO_SUBMODULE "Build and use libmbedcrypto from the crypto submodule." OFF)
+option(USE_CRYPTO_SUBMODULE "Build and use libmbedcrypto from the crypto submodule." ON)
 
 option(UNSAFE_BUILD "Allow unsafe builds. These builds ARE NOT SECURE." OFF)
 

Makefile

@@ -1,4 +1,7 @@
 
+# build crypto form submodule unless explicitly disabled
+USE_CRYPTO_SUBMODULE ?= 1
+
 DESTDIR=/usr/local
 PREFIX=mbedtls_
 
@@ -31,7 +34,7 @@ install: no_test
 	mkdir -p $(DESTDIR)/lib
 	cp -RP library/libmbedtls.*    $(DESTDIR)/lib
 	cp -RP library/libmbedx509.*   $(DESTDIR)/lib
-ifdef USE_CRYPTO_SUBMODULE
+ifneq ($(USE_CRYPTO_SUBMODULE), 0)
 	mkdir -p $(DESTDIR)/include/psa
 	cp -rp crypto/include/psa $(DESTDIR)/include
 	cp -RP crypto/library/libmbedcrypto.* $(DESTDIR)/lib
@@ -53,7 +56,7 @@ uninstall:
 	rm -f $(DESTDIR)/lib/libmbedtls.*
 	rm -f $(DESTDIR)/lib/libmbedx509.*
 	rm -f $(DESTDIR)/lib/libmbedcrypto.*
-ifdef USE_CRYPTO_SUBMODULE
+ifneq ($(USE_CRYPTO_SUBMODULE), 0)
 	$(MAKE) -C crypto uninstall
 endif
 
@@ -97,7 +100,7 @@ clean:
 	$(MAKE) -C library clean
 	$(MAKE) -C programs clean
 	$(MAKE) -C tests clean
-ifdef USE_CRYPTO_SUBMODULE
+ifneq ($(USE_CRYPTO_SUBMODULE), 0)
 	$(MAKE) -C crypto clean
 endif
 ifndef WINDOWS

README.md

@@ -21,6 +21,23 @@ The main systems used for development are CMake and GNU Make. Those systems are
 
 The Make and CMake build systems create three libraries: libmbedcrypto, libmbedx509, and libmbedtls. Note that libmbedtls depends on libmbedx509 and libmbedcrypto, and libmbedx509 depends on libmbedcrypto. As a result, some linkers will expect flags to be in a specific order, for example the GNU linker wants `-lmbedtls -lmbedx509 -lmbedcrypto`. Also, when loading shared libraries using dlopen(), you'll need to load libmbedcrypto first, then libmbedx509, before you can load libmbedtls.
 
+### Getting files form git: the Crypto submodule
+
+The Mbed Crypto library now has its own git repository, which the Mbed TLS build systems are using as a git submodule in order to build libmbecrypto as a subproject of Mbed TLS. When cloning the Mbed TLS repository, you need to make sure you're getting the submodule as well:
+
+        git clone --recursive https://github.com/ARMmbed/mbedtls.git
+
+Alternatively, if you already have an existing clone of the Mbed TLS
+repository, you can initialise and update the submodule with:
+
+        git submodule update --init crypto
+
+After these steps, your clone is now ready for building the libraries as detailed in the following sections.
+
+Note that building libmbedcrypto as a subproject of Mbed LTS does not enable the PSA-specific tests and utility programs. To use these programs, build Mbed Crypto as a standalone project.
+
+Please note that for now, Mbed TLS can only use versions of libmbedcrypto that were built as a subproject of Mbed TLS, not versions that were build standalone from the Mbed Crypto repository. This restriction will be removed in the future.
+
 ### Make
 
 We require GNU Make. To build the library and the sample programs, GNU Make and a C compiler are sufficient. Some of the more advanced build targets require some Unix/Linux tools.
@@ -158,42 +175,6 @@ Configurations
 
 We provide some non-standard configurations focused on specific use cases in the `configs/` directory. You can read more about those in `configs/README.txt`
 
-Using Mbed Crypto as a submodule
---------------------------------
-
-As an experimental feature, you can use Mbed Crypto as the source of the cryptography implementation, with Mbed TLS providing the X.509 and TLS parts of the library. Mbed Crypto is currently provided for evaluation only and should not be used in production. At this point, you should only use this option if you want to try out the experimental PSA Crypto API.
-
-To enable the use of Mbed Crypto as a submodule:
-
-1. Check out the `crypto` submodule and update it.
-
-        git submodule init crypto
-        git submodule update crypto
-
-2. (Optional) TO enable the PSA Crypto API, set the build configuration option `MBEDTLS_PSA_CRYPTO_C`. You can either edit `include/mbedtls/config.h` directly or use the configuration script:
-
-        scripts/config.pl set MBEDTLS_PSA_CRYPTO_C
-
-3. Activate the build option `USE_CRYPTO_SUBMODULE`. With GNU make, set `USE_CRYPTO_SUBMODULE=1` on each make invocation:
-
-        make USE_CRYPTO_SUBMODULE=1
-        make USE_CRYPTO_SUBMODULE=1 test
-        tests/ssl-opt.sh -f Default
-
-   Note that you need to pass `USE_CRYPTO_SUBMODULE=1` even to `make clean`. For example, if you change `config.h`, run this before rebuilding:
-
-        make USE_CRYPTO_SUBMODULE=1 clean
-
-   With CMake, create a build directory (recommended) and pass `-DUSE_CRYPTO_SUBMODULE=1` to `cmake`:
-
-        mkdir build
-        cd build
-        cmake -DUSE_CRYPTO_SUBMODULE=1 ..
-        make
-        make test
-        tests/ssl-opt.sh -f Default
-
-Note that this does not enable the PSA-specific tests and utility programs. To use these programs, use Mbed Crypto as a standalone project.
 
 Porting Mbed TLS
 ----------------

library/Makefile

@@ -1,6 +1,9 @@
 
 # Also see "include/mbedtls/config.h"
 
+# build crypto form submodule unless explicitly disabled
+USE_CRYPTO_SUBMODULE ?= 1
+
 CFLAGS	?= -O2
 WARNING_CFLAGS ?=  -Wall -W -Wdeclaration-after-statement
 LDFLAGS ?=
@@ -64,7 +67,7 @@ endif
 endif
 
 
-ifdef USE_CRYPTO_SUBMODULE
+ifneq ($(USE_CRYPTO_SUBMODULE), 0)
 # Look in crypto for libmbedcrypto.
 LOCAL_LDFLAGS += -L../crypto/library
 LOCAL_CFLAGS += -I../crypto/include
@@ -174,7 +177,7 @@ libmbedx509.dll: $(OBJS_X509) libmbedcrypto.dll
 	$(CC) -shared -Wl,-soname,$@ -Wl,--out-implib,$@.a -o $@ $(OBJS_X509) -lws2_32 -lwinmm -lgdi32 -L. -lmbedcrypto -static-libgcc $(LOCAL_LDFLAGS) $(LDFLAGS)
 
 # crypto
-ifdef USE_CRYPTO_SUBMODULE
+ifneq ($(USE_CRYPTO_SUBMODULE), 0)
 libmbedcrypto.%:
 	$(MAKE) CRYPTO_INCLUDES:="-I../../include -I../include" -C ../crypto/library $@
 else

programs/Makefile

@@ -1,4 +1,7 @@
 
+# build crypto form submodule unless explicitly disabled
+USE_CRYPTO_SUBMODULE ?= 1
+
 # To compile on SunOS: add "-lsocket -lnsl" to LDFLAGS
 # To compile with PKCS11: add "-lpkcs11-helper" to LDFLAGS
 
@@ -14,7 +17,7 @@ LOCAL_LDFLAGS = -L../library 			\
 		-lmbedx509$(SHARED_SUFFIX)	\
 		-lmbedcrypto$(SHARED_SUFFIX)
 
-ifdef USE_CRYPTO_SUBMODULE
+ifneq ($(USE_CRYPTO_SUBMODULE), 0)
 LOCAL_LDFLAGS += -L../crypto/library
 LOCAL_CFLAGS += -I../crypto/include
 LOCAL_CXXFLAGS += -I../crypto/include

tests/Makefile

@@ -1,4 +1,7 @@
 
+# build crypto form submodule unless explicitly disabled
+USE_CRYPTO_SUBMODULE ?= 1
+
 # To compile on SunOS: add "-lsocket -lnsl" to LDFLAGS
 # To compile with PKCS11: add "-lpkcs11-helper" to LDFLAGS
 
@@ -12,7 +15,7 @@ LOCAL_LDFLAGS = -L../library			\
 		-lmbedx509$(SHARED_SUFFIX)	\
 		-lmbedcrypto$(SHARED_SUFFIX)
 
-ifdef USE_CRYPTO_SUBMODULE
+ifneq ($(USE_CRYPTO_SUBMODULE), 0)
 LOCAL_LDFLAGS += -L../crypto/library
 LOCAL_CFLAGS += -I../crypto/include
 CRYPTO := ../crypto/library/

tests/scripts/all.sh

@@ -769,7 +769,6 @@ component_build_deprecated () {
     make CC=clang CFLAGS='-O -Werror -Wall -Wextra -Wno-unused-function' tests
 }
 
-
 component_test_depends_curves () {
     msg "test/build: curves.pl (gcc)" # ~ 4 min
     record_status tests/scripts/curves.pl
@@ -802,11 +801,10 @@ component_build_default_make_gcc_and_cxx () {
 }
 
 component_test_submodule_cmake () {
-    # USE_CRYPTO_SUBMODULE: check that the build works with CMake
+    # USE_CRYPTO_SUBMODULE: check that it's enabled by default with cmake
-    msg "build: cmake, full config + USE_CRYPTO_SUBMODULE, gcc+debug"
+    msg "build: cmake, full config (with USE_CRYPTO_SUBMODULE), gcc+debug"
     scripts/config.pl full # enables md4 and submodule doesn't enable md4
-    scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests
+    CC=gcc cmake -D CMAKE_BUILD_TYPE=Debug .
-    CC=gcc cmake -D USE_CRYPTO_SUBMODULE=1 -D CMAKE_BUILD_TYPE=Debug .
     make
     msg "test: top-level libmbedcrypto wasn't built (USE_CRYPTO_SUBMODULE, cmake)"
     if_build_succeeded not test -f library/libmbedcrypto.a
@@ -814,51 +812,67 @@ component_test_submodule_cmake () {
     if_build_succeeded objdump -g crypto/library/libmbedcrypto.a | grep -E 'crypto/library$' > /dev/null
     msg "test: libmbedcrypto uses top-level config (USE_CRYPTO_SUBMODULE, cmake)"
     if_build_succeeded objdump -g crypto/library/libmbedcrypto.a | grep 'md4.c' > /dev/null
-    msg "test: main suites (USE_CRYPTO_SUBMODULE, cmake)"
+    # no need for runtime tests - this is the default, tested elsewhere
-    make test
+
-    msg "test: ssl-opt.sh (USE_CRYPTO_SUBMODULE, cmake)"
-    if_build_succeeded tests/ssl-opt.sh
 }
 
 component_test_submodule_make () {
-    # USE_CRYPTO_SUBMODULE: check that the build works with make
+    # USE_CRYPTO_SUBMODULE: check that it's enabled by default with make
-    msg "build: make, full config + USE_CRYPTO_SUBMODULE, gcc+debug"
+    msg "build: make, full config (with USE_CRYPTO_SUBMODULE), gcc+debug"
     scripts/config.pl full # enables md4 and submodule doesn't enable md4
-    scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests
     make CC=gcc CFLAGS='-g' USE_CRYPTO_SUBMODULE=1
+
     msg "test: top-level libmbedcrypto wasn't built (USE_CRYPTO_SUBMODULE, make)"
     if_build_succeeded not test -f library/libmbedcrypto.a
+
     msg "test: libmbedcrypto symbols are from crypto files (USE_CRYPTO_SUBMODULE, make)"
     if_build_succeeded objdump -g crypto/library/libmbedcrypto.a | grep -E 'crypto/library$' > /dev/null
+
     msg "test: libmbedcrypto uses top-level config (USE_CRYPTO_SUBMODULE, make)"
     if_build_succeeded objdump -g crypto/library/libmbedcrypto.a | grep 'md4.c' > /dev/null
-    msg "test: main suites (USE_CRYPTO_SUBMODULE, make)"
-    make CC=gcc USE_CRYPTO_SUBMODULE=1 test
-    msg "test: ssl-opt.sh (USE_CRYPTO_SUBMODULE, make)"
-    if_build_succeeded tests/ssl-opt.sh
 }
 
 component_test_not_submodule_make () {
-    # Don't USE_CRYPTO_SUBMODULE: check that the submodule is not used with make
+    # Disable USE_CRYPTO_SUBMODULE: check that the submodule is not used with make
-    msg "build: make, full config - USE_CRYPTO_SUBMODULE, gcc+debug"
+    msg "build: make, full config without USE_CRYPTO_SUBMODULE, gcc+debug"
     scripts/config.pl full
-    make CC=gcc CFLAGS='-g'
+    scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests
-    msg "test: submodule libmbedcrypto wasn't built (USE_CRYPTO_SUBMODULE, make)"
+    make CC=gcc CFLAGS='-g' USE_CRYPTO_SUBMODULE=0
+
+    msg "test: submodule libmbedcrypto wasn't built (no USE_CRYPTO_SUBMODULE, make)"
     if_build_succeeded not test -f crypto/library/libmbedcrypto.a
-    msg "test: libmbedcrypto symbols are from library files (USE_CRYPTO_SUBMODULE, make)"
+
+    msg "test: libmbedcrypto symbols are from library files (no USE_CRYPTO_SUBMODULE, make)"
     if_build_succeeded objdump -g library/libmbedcrypto.a | grep -E 'library$' | not grep 'crypto' > /dev/null
+
+    # run some tests to validate this non-default build
+    msg "test: main suites (no USE_CRYPTO_SUBMODULE, cmake)"
+    make test
+
+    msg "test: ssl-opt.sh (no USE_CRYPTO_SUBMODULE, cmake)"
+    if_build_succeeded tests/ssl-opt.sh
 }
 
 component_test_not_submodule_cmake () {
     # Don't USE_CRYPTO_SUBMODULE: check that the submodule is not used with CMake
-    msg "build: cmake, full config - USE_CRYPTO_SUBMODULE, gcc+debug"
+    msg "build: cmake, full config without USE_CRYPTO_SUBMODULE, gcc+debug"
     scripts/config.pl full
-    CC=gcc cmake -D CMAKE_BUILD_TYPE=Debug .
+    scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests
+    CC=gcc cmake -D CMAKE_BUILD_TYPE=Debug -D USE_CRYPTO_SUBMODULE=Off .
     make
-    msg "test: submodule libmbedcrypto wasn't built (USE_CRYPTO_SUBMODULE, cmake)"
+
+    msg "test: submodule libmbedcrypto wasn't built (no USE_CRYPTO_SUBMODULE, cmake)"
     if_build_succeeded not test -f crypto/library/libmbedcrypto.a
-    msg "test: libmbedcrypto symbols are from library files (USE_CRYPTO_SUBMODULE, cmake)"
+
+    msg "test: libmbedcrypto symbols are from library files (no USE_CRYPTO_SUBMODULE, cmake)"
     if_build_succeeded objdump -g library/libmbedcrypto.a | grep -E 'library$' | not grep 'crypto' > /dev/null
+
+    # run some tests to validate this non-default build
+    msg "test: main suites (no USE_CRYPTO_SUBMODULE, cmake)"
+    make test
+
+    msg "test: ssl-opt.sh (no USE_CRYPTO_SUBMODULE, cmake)"
+    if_build_succeeded tests/ssl-opt.sh
 }
 
 component_test_use_psa_crypto_full_cmake_asan() {
@@ -869,7 +883,7 @@ component_test_use_psa_crypto_full_cmake_asan() {
     scripts/config.pl unset MBEDTLS_ECP_RESTARTABLE  # restartable ECC not supported through PSA
     scripts/config.pl set MBEDTLS_PSA_CRYPTO_C
     scripts/config.pl set MBEDTLS_USE_PSA_CRYPTO
-    CC=gcc cmake -D USE_CRYPTO_SUBMODULE=1 -D CMAKE_BUILD_TYPE:String=Asan .
+    CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
     make
 
     msg "test: main suites (MBEDTLS_USE_PSA_CRYPTO)"